diff options
Diffstat (limited to 'doc')
620 files changed, 11871 insertions, 11354 deletions
diff --git a/doc/.vale/gitlab/Admin.yml b/doc/.vale/gitlab/Admin.yml index 987dbfdbd09..f6b0a988499 100644 --- a/doc/.vale/gitlab/Admin.yml +++ b/doc/.vale/gitlab/Admin.yml @@ -10,4 +10,4 @@ link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html level: suggestion ignorecase: false swap: - '[Aa]dmin ?\w*': '(?:Admin( Area| Mode)?|[Aa]dminist(ration|rator|rators|er|rative))' + '[Aa]dmin ?\w*': '(?:Admin( Area| Mode)?|[Aa]dminist(ration|rator|rators|er|rative|ering|ered))' diff --git a/doc/.vale/gitlab/BadPlurals.yml b/doc/.vale/gitlab/BadPlurals.yml index 81e2a0e563d..efc55ac3d56 100644 --- a/doc/.vale/gitlab/BadPlurals.yml +++ b/doc/.vale/gitlab/BadPlurals.yml @@ -11,4 +11,4 @@ level: warning scope: raw ignorecase: true raw: - - '\w*\(s\)(?<!http\(s\))' + - '\b\w+\(s\)(?<!http\(s\))' diff --git a/doc/.vale/gitlab/CIConfigFile.yml b/doc/.vale/gitlab/CIConfigFile.yml new file mode 100644 index 00000000000..5a65e51ea16 --- /dev/null +++ b/doc/.vale/gitlab/CIConfigFile.yml @@ -0,0 +1,17 @@ +--- +# Error: gitlab.CIConfigFile +# +# Checks that the `.gitlab-ci.yml` file is referenced properly. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'The CI/CD configuration file should be exactly: `.gitlab-ci.yml`' +link: https://docs.gitlab.com/ee/development/documentation/versions.html +level: error +scope: raw +raw: + - '(`gitlab-ci.yml`|' + - '`gitlabci.yml`|' + - '`gitlab.ci.yml`|' + - '`.gitlab.ci-yml`|' + - '`.gitlab-ci.yaml`)' diff --git a/doc/.vale/gitlab/SubstitutionSuggestions.yml b/doc/.vale/gitlab/SubstitutionSuggestions.yml index df132d89637..4b77def065d 100644 --- a/doc/.vale/gitlab/SubstitutionSuggestions.yml +++ b/doc/.vale/gitlab/SubstitutionSuggestions.yml @@ -7,7 +7,7 @@ # For a list of all options, see https://errata-ai.github.io/vale/styles/ extends: substitution message: 'Consider %s instead of "%s".' -link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#language +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html level: suggestion ignorecase: true swap: @@ -15,7 +15,7 @@ swap: active users: '"billable users"' docs: '"documentation"' e-mail: '"email"' - GFM: '"GitLab Flavored Markdown"' + GLFM: '"GitLab Flavored Markdown"' it is recommended: '"we recommend"' navigate: go OAuth2: '"OAuth 2.0"' diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 7b20887f53f..d17015b97fd 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -12,6 +12,7 @@ level: warning ignorecase: true swap: air(?:-| )?gapped: offline environment + bullet: list item click: select code base: codebase config: configuration @@ -20,7 +21,9 @@ swap: distro: distribution file name: filename filesystem: file system + GFM: GLFM info: information + n/a: not applicable repo: repository timezone: time zone utilize: use diff --git a/doc/.vale/gitlab/VersionText.yml b/doc/.vale/gitlab/VersionText.yml index fbdda17e2a0..68753de60aa 100644 --- a/doc/.vale/gitlab/VersionText.yml +++ b/doc/.vale/gitlab/VersionText.yml @@ -16,7 +16,7 @@ # For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence message: 'This introduced-in line is not formatted correctly.' -link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#version-text-in-the-version-history +link: https://docs.gitlab.com/ee/development/documentation/versions.html level: error scope: raw raw: diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 76e8f99b5db..92e10a91753 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -25,6 +25,7 @@ arity Artifactory Asana Asciidoctor +asdf Assembla Atlassian auditability @@ -64,6 +65,7 @@ Bamboo Bazel Bhyve Bitbucket +Bitnami blockquote blockquoted blockquotes @@ -93,8 +95,13 @@ Camo canonicalization canonicalized captcha +Casdoor CentOS +Ceph Certbot +cgo +cgroup +cgroups chai changeset changesets @@ -104,11 +111,13 @@ chatbots ChatOps checksummed checksumming +Chemlab Citrix Citus clonable Cloudwatch Cobertura +Codeception Codepen Cognito colocated @@ -174,6 +183,7 @@ disambiguates discoverability dismissable Disqus +Distroless Divio Dockerfile Dockerfiles @@ -199,6 +209,7 @@ enums ETag Excon exfiltration +ExifTool expirable Facebook failover @@ -206,6 +217,7 @@ failovers failsafe Falco falsy +Fargate fastlane Fastzip favicon @@ -254,7 +266,9 @@ Gradle Grafana Grafonnet gravatar +Grype Gzip +Hackathon Haml hardcode hardcoded @@ -333,6 +347,7 @@ Leiningen libFuzzer Libravatar liveness +lockfiles Lodash Lograge logrotate @@ -352,6 +367,7 @@ Makefile Makefiles Markdown markdownlint +Marketo matcher matchers Matomo @@ -364,6 +380,7 @@ memoizing Memorystore mergeability mergeable +metaprogramming Microsoft middleware middlewares @@ -386,6 +403,7 @@ ModSecurity Monokai monorepo monorepos +monospace multiline mutex nameserver @@ -399,6 +417,7 @@ Nanoc negatable Netlify Nokogiri +nosniff noteable noteables npm @@ -472,6 +491,7 @@ proxies proxyable proxying pseudocode +pseudonymization pseudonymized pseudonymizer Puma @@ -484,6 +504,7 @@ Rackspace Raspbian rbenv rbtrace +Rclone Rdoc reachability Realplayer @@ -575,6 +596,7 @@ serializer serializers serializing serverless +setuptools severities sharded sharding @@ -592,10 +614,12 @@ Slony smartcard smartcards snapshotting +Snyk Sobelow Solargraph Solarized Sourcegraph +Spamcheck spammable sparkline sparklines @@ -660,6 +684,7 @@ Sysbench syscall syscalls syslog +systemd tanuki tcpdump templated @@ -700,6 +725,7 @@ Twilio Twitter TypeScript Ubuntu +Udemy unapplied unapprove unapproved @@ -804,6 +830,7 @@ Vagrantfile validator validators vendored +vendoring versionless viewport viewports @@ -828,6 +855,7 @@ wireframes wireframing Wireshark Wordpress +Workato worktree worktrees Worldline @@ -836,6 +864,7 @@ Xeon YouTrack ytt Yubico +Zabbix Zeitwerk Zendesk ZenTao diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 6cf630ad079..07979e21038 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -132,13 +132,11 @@ Delete an event streaming destination by specifying an ID. Get the required ID b streaming destinations. ```graphql - -mutation{ +mutation { externalAuditEventDestinationDestroy(input: { id: destination }) { errors } } - ``` Destination is deleted if: @@ -158,10 +156,11 @@ the destination's value when [listing streaming destinations](#list-streaming-de ## Audit event streaming on Git operations -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.0. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. On GitLab.com, this feature is not available. +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. On GitLab.com, this feature is available. Streaming audit events can be sent when signed-in users push or pull a project's remote Git repositories: diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 36f3a71cd2e..6fac21ef889 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -10,6 +10,7 @@ GitLab offers a way to view the changes made within the GitLab server for owners on a [paid plan](https://about.gitlab.com/pricing/). GitLab system administrators can also view all audit events by accessing the [`audit_json.log` file](logs.md#audit_jsonlog). +The JSON audit log does not include events that are [only streamed](../development/audit_event_guide/index.md#event-streaming). You can: @@ -33,7 +34,7 @@ permission level, who added a new user, or who removed a user. ## Retention policy There is no retention policy in place for audit events. -See the [Specify a retention period for audit events](https://gitlab.com/gitlab-org/gitlab/-/issues/8137) for more information. +See the [Specify a retention period for audit events](https://gitlab.com/groups/gitlab-org/-/epics/7917) for more information. ## List of events @@ -114,6 +115,8 @@ From there, you can see the following actions: - Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. - Group deploy token was successfully created, revoked, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. - Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. +- [IP restrictions](../user/group/index.md#restrict-group-access-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0 +- Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events) @@ -178,6 +181,11 @@ From there, you can see the following actions: - Skipped pipelines are considered successful enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) - All discussions must be resolved enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) - Commit message suggestion is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9) +- Status check is added, edited, or deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) +- Merge commit message template is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) +- Squash commit message template is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) +- Default description template for merge requests is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0) +- Project was scheduled for deletion due to inactivity ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0) Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events). @@ -258,36 +266,15 @@ Don't see the event you want in any of the epics linked above? You can either: request it. - [Add it yourself](../development/audit_event_guide/). -### Disabled events +### Removed events -#### Repository push (DEPRECATED) +> - Repositories push events was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. +> - Repositories push events was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 15.0. -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. +The repositories push events feature was: -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. - -The current architecture of audit events is not prepared to receive a very high amount of records. -It may make the user interface for your project or audit events very busy, and the disk space consumed by the -`audit_events` PostgreSQL table may increase considerably. It's disabled by default -to prevent performance degradations on GitLab instances with very high Git write traffic. - -If you still wish to enable **Repository push** events in your instance, follow -the steps below. - -**In Omnibus installations:** - -1. Enter the Rails console: - - ```shell - sudo gitlab-rails console - ``` - -1. Flip the switch and enable the feature flag: - - ```ruby - Feature.enable(:repository_push_audit_event) - ``` +- [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 14.3. +- [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337993) in GitLab 15.0. ## Search diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index b3304fd1cbd..1d0aff51a04 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -6,88 +6,52 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Auditor users **(PREMIUM SELF)** -Auditor users are given read-only access to all projects, groups, and other -resources on the GitLab instance. - -## Overview - -Auditor users are able to have both full access to their own resources -(including projects, groups, and snippets) and read-only access to _all_ other -resources, except the [Admin Area](../user/admin_area/index.md). These user -accounts are regular users who can be added to projects, create personal -snippets, and create milestones on their groups, while also having read-only -access to all projects on the server to which they haven't been explicitly -[given access](../user/permissions.md). - -The `Auditor` access level is _not_ a read-only version of the `Admin` access level. Auditor users -can't access the project or group settings pages, or the Admin Area. - -Assuming you have signed in as an Auditor user: - -- For a project the Auditor is not member of, the Auditor should have - read-only access. If the project is public or internal, they have the same - access as users that aren't members of that project or group. -- For a project the Auditor owns, the Auditor should have full access to - everything. -- For a project to which the Auditor is added as a member, the Auditor should - have the same access as their given [permissions](../user/permissions.md). - For example, if they were added as a Developer, they can push commits or - comment on issues. -- The Auditor can't view the Admin Area, or perform any administration actions. - -For more information about what an Auditor can or can't do, see the -[Permissions and restrictions of an Auditor user](#permissions-and-restrictions-of-an-auditor-user) -section. +Users with auditor access have read-only access to all groups, projects, and other resources except: + +- The [Admin Area](../user/admin_area/index.md). +- Project and group settings. -## Use cases +For more information, see [Auditor user permissions and restrictions](#auditor-user-permissions-and-restrictions) +section. -The following use cases describe some situations where Auditor users could be -helpful: +Situations where auditor access for users could be helpful include: - Your compliance department wants to run tests against the entire GitLab base to ensure users are complying with password, credit card, and other sensitive - data policies. With Auditor users, this can be achieved very without having - to give them user administration rights or using the API to add them to all projects. + data policies. You can achieve this with auditor access without giving the compliance department + user administration rights or adding them to all projects. - If particular users need visibility or access to most of all projects in your GitLab instance, instead of manually adding the user to all projects, - you can create an Auditor user and then share the credentials with those users - to which you want to grant access. + you can create an account with auditor access and then share the credentials + with those users to which you want to grant access. -## Add an Auditor user +## Add a user with auditor access -To create an Auditor user: +To create a new user account with auditor access (or change an existing user): + +To create a user account with auditor access: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Users**. -1. Create a new user or edit an existing one, and in the **Access** section - select Auditor. +1. Create a new user or edit an existing one. Set **Access Level** to **Auditor**. 1. If you created a user, select **Create user**. For an existing user, select **Save changes**. -To revoke Auditor permissions from a user, make them a Regular user by -following the previous steps. - -Additionally users can be set as an Auditor using [SAML groups](../integration/saml.md#auditor-groups). - -## Permissions and restrictions of an Auditor user - -An Auditor user should be able to access all projects and groups of a GitLab -instance, with the following permissions and restrictions: - -- Has read-only access to the API -- Can access projects that are: - - Private - - Public - - Internal -- Can read all files in a repository -- Can read issues and MRs -- Can read project snippets -- Cannot be Administrator and Auditor at the same time -- Cannot access the Admin Area -- In a group or project they're not a member of: - - Cannot access project settings - - Cannot access group settings - - Cannot commit to repository - - Cannot create or comment on issues and MRs - - Cannot create or modify files from the Web UI - - Cannot merge a merge request - - Cannot create project snippets +To revoke auditor access from a user, follow these steps but set **Access Level** to **Regular**. + +You can also give users auditor access using [SAML groups](../integration/saml.md#auditor-groups). + +## Auditor user permissions and restrictions + +Auditor access is _not_ a read-only version of administrator access because it doesn't permit access to the Admin Area. + +For access to their own resources and resources within a group or project where they are a member, +users with auditor access have the same [permissions](../user/permissions.md) as regular users. + +If you are signed in with auditor access, you: + +- Have full access to projects you own. +- Have read-only access to projects you aren't a member of. +- Have [permissions](../user/permissions.md) based on your role to projects you are a member of. For example, if you have the Developer role, + you can push commits or comment on issues. +- Can access the same resources using the GitLab UI or API. +- Can't view the Admin Area, or perform any administration actions. diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index a7e070b755a..9232e79b800 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -468,7 +468,7 @@ If initially your LDAP configuration looked like: password: '123' ``` -1. Edit `/etc/gitlab/gitlab.rb` and remove the settings for `user_dn` and `password`. +1. Edit `/etc/gitlab/gitlab.rb` and remove the settings for `bind_dn` and `password`. 1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -502,7 +502,7 @@ If initially your LDAP configuration looked like: password: '123' ``` -1. Edit `config/gitlab.yaml` and remove the settings for `user_dn` and `password`. +1. Edit `config/gitlab.yaml` and remove the settings for `bind_dn` and `password`. 1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -531,16 +531,19 @@ However, these users can continue to use Git with SSH until the next time the To delete the account immediately, you can manually [block the user](../../../user/admin_area/moderate_users.md#block-a-user). -## Updating user email addresses +## Update user email addresses -Email addresses on the LDAP server are considered the source of truth for users when LDAP is used to sign in. Updating user email -addresses must be done on the LDAP server that manages the user. The email address for GitLab is updated either: +Email addresses on the LDAP server are considered the source of truth for users when LDAP is used to sign in. + +Updating user email addresses must be done on the LDAP server that manages the user. The email address for GitLab is updated either: - When the user next signs in. - When the next [user sync](ldap_synchronization.md#user-sync) is run. The updated user's previous email address becomes the secondary email address to preserve that user's commit history. +You can find more details on the expected behavior of user updates in our [LDAP troubleshooting section](ldap-troubleshooting.md#user-dn-orand-email-have-changed). + ## Google Secure LDAP > Introduced in GitLab 11.9. diff --git a/doc/administration/auth/ldap/ldap_synchronization.md b/doc/administration/auth/ldap/ldap_synchronization.md index d7ce54a47c0..0f0d301bfa9 100644 --- a/doc/administration/auth/ldap/ldap_synchronization.md +++ b/doc/administration/auth/ldap/ldap_synchronization.md @@ -190,8 +190,8 @@ to lock down user abilities to invite new members to a group. When enabled, the following applies: -- Only administrator can manage memberships of any group including access levels. -- Users are not allowed to share project with other groups or invite members to +- Only an administrator can manage memberships of any group including access levels. +- Users are not allowed to share a project with other groups or invite members to a project created in a group. To enable it, you must: diff --git a/doc/administration/configure.md b/doc/administration/configure.md index e42eb632f14..cba4a23d95a 100644 --- a/doc/administration/configure.md +++ b/doc/administration/configure.md @@ -30,7 +30,7 @@ The tables lists features on the left and provides their capabilities to the rig ## Geo Capabilities -If your availabity needs to span multiple zones or multiple locations, please read about [Geo](geo/index.md). +If your availability needs to span multiple zones or multiple locations, please read about [Geo](geo/index.md). | | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| |-|--------------|----------------|-----------------|-------------|-----------------| @@ -44,7 +44,7 @@ The following table outlines failure modes and mitigation paths for the product | ----------- | -------------------------- | ----------------------------- | ---------------------------------- | ----- | | Single Gitaly Node | Downtime - Must restore from backup | Downtime - Must restore from Backup | Downtime - Must wait for outage to end | | | Single Gitaly Node + Geo Secondary | Downtime - Must restore from backup, can perform a manual failover to secondary | Downtime - Must restore from Backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | -| Sharded Gitaly Install | Partial Downtime - Only repos on impacted node affected, must restore from backup | Partial Downtime - Only repos on impacted node affected, must restore from backup | Downtime - Must wait for outage to end | | -| Sharded Gitaly Install + Geo Secondary | Partial Downtime - Only repos on impacted node affected, must restore from backup, could perform manual failover to secondary for impacted repos | Partial Downtime - Only repos on impacted node affected, must restore from backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | +| Sharded Gitaly Install | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Downtime - Must wait for outage to end | | +| Sharded Gitaly Install + Geo Secondary | Partial Downtime - Only repositories on impacted node affected, must restore from backup, could perform manual failover to secondary for impacted repositories | Partial Downtime - Only repositories on impacted node affected, must restore from backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | | Gitaly Cluster Install* | No Downtime - will swap repository primary to another node after 10 seconds | N/A - All writes are voted on by multiple Gitaly Cluster nodes | Downtime - Must wait for outage to end | Snapshot backups for Gitaly Cluster nodes not supported at this time | | Gitaly Cluster Install* + Geo Secondary | No Downtime - will swap repository primary to another node after 10 seconds | N/A - All writes are voted on by multiple Gitaly Cluster nodes | Manual intervention - failover to Geo secondary | Snapshot backups for Gitaly Cluster nodes not supported at this time | diff --git a/doc/administration/encrypted_configuration.md b/doc/administration/encrypted_configuration.md index baa6e967507..5c943e56b98 100644 --- a/doc/administration/encrypted_configuration.md +++ b/doc/administration/encrypted_configuration.md @@ -11,7 +11,7 @@ type: reference GitLab can read settings for certain features from encrypted settings files. The supported features are: -- [LDAP `user_bn` and `password`](auth/ldap/index.md#use-encrypted-credentials). +- [LDAP `bind_dn` and `password`](auth/ldap/index.md#use-encrypted-credentials). - [SMTP `user_name` and `password`](raketasks/smtp.md#secrets). In order to enable the encrypted configuration settings, a new base key needs to be generated for diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index 22159b6e9db..1fe5b223f8b 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -37,31 +37,6 @@ You can use the following environment variables to override certain values: | `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | | `UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`). | -## Complete database variables - -The recommended method for specifying your database connection information is -to set the `DATABASE_URL` environment variable. This variable contains -connection information (`adapter`, `database`, `username`, `password`, `host`, -and `port`), but no behavior information (`encoding` or `pool`). If you don't -want to use `DATABASE_URL`, or want to set database behavior information, -either: - -- Copy the template file, `cp config/database.yml.env config/database.yml`. -- Set a value for some `GITLAB_DATABASE_XXX` variables. - -The list of `GITLAB_DATABASE_XXX` variables that you can set is: - -| Variable | Default value | Overridden by `DATABASE_URL`? | -|-----------------------------|--------------------------------|-------------------------------| -| `GITLAB_DATABASE_ADAPTER` | `postgresql` | **{check-circle}** Yes | -| `GITLAB_DATABASE_DATABASE` | `gitlab_#{ENV['RAILS_ENV']` | **{check-circle}** Yes | -| `GITLAB_DATABASE_ENCODING` | `unicode` | **{dotted-circle}** No | -| `GITLAB_DATABASE_HOST` | `localhost` | **{check-circle}** Yes | -| `GITLAB_DATABASE_PASSWORD` | _none_ | **{check-circle}** Yes | -| `GITLAB_DATABASE_POOL` | `10` | **{dotted-circle}** No | -| `GITLAB_DATABASE_PORT` | `5432` | **{check-circle}** Yes | -| `GITLAB_DATABASE_USERNAME` | `root` | **{check-circle}** Yes | - ## Adding more variables We welcome merge requests to make more settings configurable by using variables. diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index 64673b9ee8d..4e68cc39580 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -42,7 +42,7 @@ To bring the former **primary** site up to date: NOTE: If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record) for this site during disaster recovery procedure you may need to [block - all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-node) + all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-site) during this procedure. 1. [Set up database replication](../setup/database.md). In this case, the **secondary** site diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index 1725c283396..baece830318 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -100,15 +100,15 @@ Note the following when promoting a secondary: #### Promoting a **secondary** site running on a single node running GitLab 14.5 and later -1. SSH in to your **secondary** node and execute: +1. SSH in to your **secondary** site and execute: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -122,10 +122,10 @@ Note the following when promoting a secondary: WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. -1. SSH in to your **secondary** node and login as root: +1. SSH in to your **secondary** site and login as root: ```shell sudo -i @@ -142,7 +142,7 @@ Use `gitlab-ctl geo promote` instead. 1. Promote the **secondary** site to the **primary** site: - - To promote the secondary node to primary along with [preflight checks](planned_failover.md#preflight-checks): + - To promote the secondary site to primary along with [preflight checks](planned_failover.md#preflight-checks): ```shell gitlab-ctl promote-to-primary-node @@ -163,7 +163,7 @@ Use `gitlab-ctl geo promote` instead. shows that it is complete, you can skip the preflight checks to make the command complete promotion. This bug was fixed in GitLab 13.8 and later. - - To promote the secondary node to primary **without any further confirmation**, + - To promote the secondary site to primary **without any further confirmation**, even when preflight checks fail: ```shell @@ -178,13 +178,13 @@ Use `gitlab-ctl geo promote` instead. 1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the node on the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -192,13 +192,13 @@ Use `gitlab-ctl geo promote` instead. 1. SSH into each Rails node on your **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -212,7 +212,7 @@ Use `gitlab-ctl geo promote` instead. WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. The `gitlab-ctl promote-to-primary-node` command cannot be used yet in @@ -256,13 +256,13 @@ do this manually. 1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -270,13 +270,13 @@ do this manually. 1. SSH into each Rails node on your **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -290,7 +290,7 @@ do this manually. WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. The `gitlab-ctl promote-to-primary-node` command cannot be used yet in @@ -347,7 +347,7 @@ site first. - [Azure PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) - [Google Cloud SQL](https://cloud.google.com/sql/docs/mysql/replication/manage-replicas#promote-replica) - For other external PostgreSQL databases, save the following script in your - secondary node, for example `/tmp/geo_promote.sh`, and modify the connection + secondary site, for example `/tmp/geo_promote.sh`, and modify the connection parameters to match your environment. Then, execute it to promote the replica: ```shell @@ -370,13 +370,13 @@ site first. 1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -384,13 +384,13 @@ site first. 1. SSH into each Rails node on your **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -404,7 +404,7 @@ site first. WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. The `gitlab-ctl promote-to-primary-node` command cannot be used in conjunction with @@ -418,7 +418,7 @@ required: - [Azure PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) - [Google Cloud SQL](https://cloud.google.com/sql/docs/mysql/replication/manage-replicas#promote-replica) - For other external PostgreSQL databases, save the following script in your - secondary node, for example `/tmp/geo_promote.sh`, and modify the connection + secondary site, for example `/tmp/geo_promote.sh`, and modify the connection parameters to match your environment. Then, execute it to promote the replica: ```shell @@ -503,7 +503,7 @@ secondary domain, like changing Git remotes and API URLs. in `/etc/gitlab/gitlab.rb`. 1. For GitLab 12.0 through 12.7, you may need to update the **primary** - node's name in the database. This bug has been fixed in GitLab 12.8. + site's name in the database. This bug has been fixed in GitLab 12.8. To determine if you need to do this, search for the `gitlab_rails["geo_node_name"]` setting in your `/etc/gitlab/gitlab.rb` @@ -571,9 +571,9 @@ and after that you also need two extra steps. # Allow PostgreSQL client authentication from the primary and secondary IPs. These IPs may be # public or VPC addresses in CIDR format, for example ['198.51.100.1/32', '198.51.100.2/32'] ## - postgresql['md5_auth_cidr_addresses'] = ['<primary_node_ip>/32', '<secondary_node_ip>/32'] + postgresql['md5_auth_cidr_addresses'] = ['<primary_site_ip>/32', '<secondary_site_ip>/32'] - # Every secondary server needs to have its own slot so specify the number of secondary nodes you're going to have + # Every secondary site needs to have its own slot so specify the number of secondary sites you're going to have postgresql['max_replication_slots'] = 1 ## @@ -643,7 +643,7 @@ avoid a split-brain situation where writes can occur in two different GitLab instances, complicating recovery efforts. So to prepare for the failover, you must disable the **primary** site: -- If you have access to the **primary** Kubernetes cluster, connect to it and disable the GitLab webservice and Sidekiq pods: +- If you have access to the **primary** Kubernetes cluster, connect to it and disable the GitLab `webservice` and `Sidekiq` pods: ```shell kubectl --namespace gitlab scale deploy gitlab-geo-webservice-default --replicas=0 @@ -662,7 +662,7 @@ must disable the **primary** site: - Revoke object storage permissions from the **primary** site. - Physically disconnect a machine. -### Step 2. Promote all **secondary** sites external to the cluster +### Step 2. Promote all **secondary** site nodes external to the cluster WARNING: If the secondary site [has been paused](../../geo/index.md#pausing-and-resuming-replication), this performs @@ -673,13 +673,13 @@ If you are running GitLab 14.5 and later: 1. For each node outside of the **secondary** Kubernetes cluster using Omnibus such as PostgreSQL or Gitaly, SSH into the node and run one of the following commands: - - To promote the secondary node to primary: + - To promote the **secondary** site node external to the Kubernetes cluster to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the **secondary** site node external to the Kubernetes cluster to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -699,7 +699,7 @@ If you are running GitLab 14.5 and later: If you are running GitLab 14.4 and earlier: -1. SSH in to the database node in the **secondary** and trigger PostgreSQL to +1. SSH in to the database node in the **secondary** site and trigger PostgreSQL to promote to read-write: ```shell diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 2a21bda23d7..57bad6177d9 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -12,10 +12,10 @@ the event of unplanned outage, but it can be used in conjunction with a planned failover to migrate your GitLab instance between regions without extended downtime. -As replication between Geo nodes is asynchronous, a planned failover requires -a maintenance window in which updates to the **primary** node are blocked. The +As replication between Geo sites is asynchronous, a planned failover requires +a maintenance window in which updates to the **primary** site are blocked. The length of this window is determined by your replication capacity - once the -**secondary** node is completely synchronized with the **primary** node, the failover can occur without +**secondary** site is completely synchronized with the **primary** site, the failover can occur without data loss. This document assumes you already have a fully configured, working Geo setup. @@ -28,7 +28,7 @@ have a high degree of confidence in being able to perform them accurately. ## Not all data is automatically replicated If you are using any GitLab features that Geo [doesn't support](../replication/datatypes.md#limitations-on-replicationverification), -you must make separate provisions to ensure that the **secondary** node has an +you must make separate provisions to ensure that the **secondary** site has an up-to-date copy of any data associated with that feature. This may extend the required scheduled maintenance period significantly. @@ -36,7 +36,7 @@ A common strategy for keeping this period as short as possible for data stored in files is to use `rsync` to transfer the data. An initial `rsync` can be performed ahead of the maintenance window; subsequent `rsync`s (including a final transfer inside the maintenance window) then transfers only the -*changes* between the **primary** node and the **secondary** nodes. +*changes* between the **primary** site and the **secondary** sites. Repository-centric strategies for using `rsync` effectively can be found in the [moving repositories](../../operations/moving_repositories.md) documentation; these strategies can @@ -98,31 +98,31 @@ Doing so reduces both the length of the maintenance window, and the risk of data loss as a result of a poorly executed planned failover. In GitLab 12.4, you can optionally allow GitLab to manage replication of Object Storage for -**secondary** nodes. For more information, see [Object Storage replication](../replication/object_storage.md). +**secondary** sites. For more information, see [Object Storage replication](../replication/object_storage.md). -### Review the configuration of each **secondary** node +### Review the configuration of each **secondary** site -Database settings are automatically replicated to the **secondary** node, but the +Database settings are automatically replicated to the **secondary** site, but the `/etc/gitlab/gitlab.rb` file must be set up manually, and differs between -nodes. If features such as Mattermost, OAuth or LDAP integration are enabled -on the **primary** node but not the **secondary** node, they are lost during failover. +sites. If features such as Mattermost, OAuth or LDAP integration are enabled +on the **primary** site but not the **secondary** site, they are lost during failover. -Review the `/etc/gitlab/gitlab.rb` file for both nodes and ensure the **secondary** node -supports everything the **primary** node does **before** scheduling a planned failover. +Review the `/etc/gitlab/gitlab.rb` file for both sites and ensure the **secondary** site +supports everything the **primary** site does **before** scheduling a planned failover. ### Run system checks -Run the following on both **primary** and **secondary** nodes: +Run the following on both **primary** and **secondary** sites: ```shell gitlab-rake gitlab:check gitlab-rake gitlab:geo:check ``` -If any failures are reported on either node, they should be resolved **before** +If any failures are reported on either site, they should be resolved **before** scheduling a planned failover. -### Check that secrets match between nodes +### Check that secrets and SSH host keys match between nodes The SSH host keys and `/etc/gitlab/gitlab-secrets.json` files should be identical on all nodes. Check this by running the following on all nodes and @@ -132,8 +132,14 @@ comparing the output: sudo sha256sum /etc/ssh/ssh_host* /etc/gitlab/gitlab-secrets.json ``` -If any files differ, replace the content on the **secondary** node with the -content from the **primary** node. +If any files differ, [manually replicate GitLab secrets](../replication/configuration.md#step-1-manually-replicate-secret-gitlab-values) and [replicate SSH host keys](../replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) +to the **secondary** site as necessary. + +### Check that the correct certificates are installed for HTTPS + +This step can be safely skipped if the **primary** site and all external sites accessed by the **primary** site use public CA-issued certificates. + +If the **primary** site uses custom or self-signed TLS certificates to secure inbound connections or if the **primary** site connects to external services that use custom or self-signed certificates, the correct certificates should also be installed on the **secondary** site. Follow instructions for [using custom certificates](../replication/configuration.md#step-4-optional-using-custom-certificates) with **secondary** sites. ### Ensure Geo replication is up-to-date @@ -141,13 +147,13 @@ The maintenance window won't end until Geo replication and verification is completely finished. To keep the window as short as possible, you should ensure these processes are close to 100% as possible during active use. -On the **secondary** node: +On the **secondary** site: 1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Geo > Nodes**. +1. On the left sidebar, select **Geo > Sites**. Replicated objects (shown in green) should be close to 100%, and there should be no failures (shown in red). If a large proportion of - objects aren't yet replicated (shown in gray), consider giving the node more + objects aren't yet replicated (shown in gray), consider giving the site more time to complete  @@ -160,7 +166,7 @@ You can use the [Geo status API](../../../api/geo_nodes.md#retrieve-project-sync the reasons for failure. A common cause of replication failures is the data being missing on the -**primary** node - you can resolve these failures by restoring the data from backup, +**primary** site - you can resolve these failures by restoring the data from backup, or removing references to the missing data. ### Verify the integrity of replicated data @@ -169,21 +175,21 @@ This [content was moved to another location](background_verification.md). ### Notify users of scheduled maintenance -On the **primary** node: +On the **primary** site: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Messages**. 1. Add a message notifying users on the maintenance window. - You can check under **Geo > Nodes** to estimate how long it + You can check under **Geo > Sites** to estimate how long it takes to finish syncing. 1. Select **Add broadcast message**. -## Prevent updates to the **primary** node +## Prevent updates to the **primary** site To ensure that all data is replicated to a secondary site, updates (write requests) need to be disabled on the **primary** site: -1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** node. +1. Enable [maintenance mode](../../maintenance_mode/index.md) on the **primary** site. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Monitoring > Background Jobs**. 1. On the Sidekiq dashboard, select **Cron**. @@ -199,22 +205,22 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary 1. If you are manually replicating any data not managed by Geo, trigger the final replication process now. -1. On the **primary** node: +1. On the **primary** site: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Monitoring > Background Jobs**. 1. On the Sidekiq dashboard, select **Queues**, and wait for all queues except those with `geo` in the name to drop to 0. These queues contain work that has been submitted by your users; failing over before it is completed, causes the work to be lost. - 1. On the left sidebar, select **Geo > Nodes** and wait for the - following conditions to be true of the **secondary** node you are failing over to: + 1. On the left sidebar, select **Geo > Sites** and wait for the + following conditions to be true of the **secondary** site you are failing over to: - All replication meters reach 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. - Database replication lag is 0ms. - The Geo log cursor is up to date (0 events behind). -1. On the **secondary** node: +1. On the **secondary** site: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Monitoring > Background Jobs**. 1. On the Sidekiq dashboard, select **Queues**, and wait for all the `geo` @@ -222,16 +228,16 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary 1. [Run an integrity check](../../raketasks/check.md) to verify the integrity of CI artifacts, LFS objects, and uploads in file storage. -At this point, your **secondary** node contains an up-to-date copy of everything the -**primary** node has, meaning nothing was lost when you fail over. +At this point, your **secondary** site contains an up-to-date copy of everything the +**primary** site has, meaning nothing was lost when you fail over. -## Promote the **secondary** node +## Promote the **secondary** site -After the replication is finished, [promote the **secondary** node to a **primary** node](index.md). This process causes a brief outage on the **secondary** node, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead. +After the replication is finished, [promote the **secondary** site to a **primary** site](index.md). This process causes a brief outage on the **secondary** site, and users may need to log in again. If you follow the steps correctly, the old primary Geo site should still be disabled and user traffic should go to the newly-promoted site instead. -When the promotion is completed, the maintenance window is over, and your new **primary** node now +When the promotion is completed, the maintenance window is over, and your new **primary** site now begins to diverge from the old one. If problems do arise at this point, failing -back to the old **primary** node [is possible](bring_primary_back.md), but likely to result +back to the old **primary** site [is possible](bring_primary_back.md), but likely to result in the loss of any data uploaded to the new **primary** in the meantime. Don't forget to remove the broadcast message after the failover is complete. diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 9b3e0ecb427..59134df8e7f 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -69,7 +69,7 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary On the **secondary** site: 1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Geo > Nodes** to see its status. +1. On the left sidebar, select **Geo > Sites** to see its status. Replicated objects (shown in green) should be close to 100%, and there should be no failures (shown in red). If a large proportion of objects aren't yet replicated (shown in gray), consider giving the site more @@ -84,7 +84,7 @@ failed to replicate is **lost**. You can use the [Geo status API](../../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) to review failed objects and the reasons for failure. -A common cause of replication failures is the data being missing on the +A common cause of replication failures is data that is missing on the **primary** site - you can resolve these failures by restoring the data from backup, or removing references to the missing data. @@ -100,22 +100,22 @@ follow these steps to avoid unnecessary data loss: **primary**. Your **secondary** site still needs read-only access to the **primary** site during the maintenance window: - 1. At the scheduled time, using your cloud provider or your node's firewall, block - all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and - the **secondary** node's IP. + 1. At the scheduled time, using your cloud provider or your site's firewall, block + all HTTP, HTTPS and SSH traffic to/from the **primary** site, **except** for your IP and + the **secondary** site's IP. - For instance, you can run the following commands on the **primary** node: + For instance, you can run the following commands on the **primary** site: ```shell - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 22 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT sudo iptables -A INPUT --destination-port 22 -j REJECT - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 80 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT sudo iptables -A INPUT --tcp-dport 80 -j REJECT - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 443 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT sudo iptables -A INPUT --tcp-dport 443 -j REJECT ``` @@ -157,8 +157,8 @@ follow these steps to avoid unnecessary data loss: those with `geo` in the name to drop to 0. These queues contain work that has been submitted by your users; failing over before it is completed, causes the work to be lost. - 1. On the left sidebar, select **Geo > Nodes** and wait for the - following conditions to be true of the **secondary** node you are failing over to: + 1. On the left sidebar, select **Geo > Sites** and wait for the + following conditions to be true of the **secondary** site you are failing over to: - All replication meters reach 100% replicated, 0% failures. - All verification meters reach 100% verified, 0% failures. @@ -230,13 +230,13 @@ follow these steps to avoid unnecessary data loss: 1. SSH to every Sidekiq, PostgresSQL, and Gitaly node in the **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -244,13 +244,13 @@ follow these steps to avoid unnecessary data loss: 1. SSH into each Rails node on your **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -264,7 +264,7 @@ follow these steps to avoid unnecessary data loss: WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. NOTE: diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index fa30b11fe32..25f6999ce43 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -85,22 +85,22 @@ follow these steps to avoid unnecessary data loss: **primary**. Your **secondary** site still needs read-only access to the **primary** site during the maintenance window: - 1. At the scheduled time, using your cloud provider or your node's firewall, block - all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and - the **secondary** node's IP. + 1. At the scheduled time, using your cloud provider or your site's firewall, block + all HTTP, HTTPS and SSH traffic to/from the **primary** site, **except** for your IP and + the **secondary** site's IP. - For instance, you can run the following commands on the **primary** node: + For instance, you can run the following commands on the **primary** site: ```shell - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 22 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT sudo iptables -A INPUT --destination-port 22 -j REJECT - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 80 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT sudo iptables -A INPUT --tcp-dport 80 -j REJECT - sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <secondary_site_ip> --destination-port 443 -j ACCEPT sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT sudo iptables -A INPUT --tcp-dport 443 -j REJECT ``` @@ -120,7 +120,7 @@ follow these steps to avoid unnecessary data loss: 1. On the **primary** site: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Monitoring > Background Jobs**. - 1. On the Sidekiq dhasboard, select **Cron**. + 1. On the Sidekiq dashboard, select **Cron**. 1. Select `Disable All` to disable any non-Geo periodic background jobs. 1. Select `Enable` for the `geo_sidekiq_cron_config_worker` cron job. This job re-enables several other cron jobs that are essential for planned @@ -142,7 +142,7 @@ follow these steps to avoid unnecessary data loss: those with `geo` in the name to drop to 0. These queues contain work that has been submitted by your users; failing over before it is completed, causes the work to be lost. - 1. On the left sidebar, select **Geo > Nodes** and wait for the + 1. On the left sidebar, select **Geo > Sites** and wait for the following conditions to be true of the **secondary** site you are failing over to: - All replication meters reach 100% replicated, 0% failures. @@ -224,15 +224,15 @@ Note the following when promoting a secondary: To promote the secondary site running GitLab 14.5 and later: -1. SSH in to your **secondary** node and run one of the following commands: +1. SSH in to your **secondary** site and run one of the following commands: - - To promote the secondary node to primary: + - To promote the secondary site to primary: ```shell sudo gitlab-ctl geo promote ``` - - To promote the secondary node to primary **without any further confirmation**: + - To promote the secondary site to primary **without any further confirmation**: ```shell sudo gitlab-ctl geo promote --force @@ -247,7 +247,7 @@ To promote the secondary site running GitLab 14.4 and earlier: WARNING: The `gitlab-ctl promote-to-primary-node` and `gitlab-ctl promoted-db` commands are -deprecated in GitLab 14.5 and later, and are scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +deprecated in GitLab 14.5 and later, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). Use `gitlab-ctl geo promote` instead. 1. SSH in to your **secondary** site and login as root: diff --git a/doc/administration/geo/glossary.md b/doc/administration/geo/glossary.md index a18e78a5e01..c6b3f26dc67 100644 --- a/doc/administration/geo/glossary.md +++ b/doc/administration/geo/glossary.md @@ -26,9 +26,9 @@ these definitions yet. | Single-node site | A specific configuration of GitLab that uses exactly one node. | GitLab | single-server, single-instance | Multi-node site | A specific configuration of GitLab that uses more than one node. | GitLab | multi-server, multi-instance, high availability | | Primary site | A GitLab site whose data is being replicated by at least one secondary site. There can only be a single primary site. | Geo-specific | Geo deployment, Primary node | -| Secondary site(s) | A GitLab site that is configured to replicate the data of a primary site. There can be one or more secondary sites. | Geo-specific | Geo deployment, Secondary node | +| Secondary site | A GitLab site that is configured to replicate the data of a primary site. There can be one or more secondary sites. | Geo-specific | Geo deployment, Secondary node | | Geo deployment | A collection of two or more GitLab sites with exactly one primary site being replicated by one or more secondary sites. | Geo-specific | | -| Reference architecture(s) | A [specified configuration of GitLab for a number of users](../reference_architectures/index.md), possibly including multiple nodes and multiple sites. | GitLab | | +| Reference architecture | A [specified configuration of GitLab for a number of users](../reference_architectures/index.md), possibly including multiple nodes and multiple sites. | GitLab | | | Promoting | Changing the role of a site from secondary to primary. | Geo-specific | | | Demoting | Changing the role of a site from primary to secondary. | Geo-specific | | | Failover | The entire process that shifts users from a primary Site to a secondary site. This includes promoting a secondary, but contains other parts as well. For example, scheduling maintenance. | Geo-specific | | diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 1b80e91c9c4..dc25be825ea 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -254,7 +254,7 @@ Omnibus GitLab-managed database. External databases are not supported. In some circumstances, like during [upgrades](replication/updating_the_geo_sites.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. -Pausing and resuming replication is done via a command line tool from the a node in the secondary site where the `postgresql` service is enabled. +Pausing and resuming replication is done via a command line tool from the node in the secondary site where the `postgresql` service is enabled. If `postgresql` is on a standalone database node, ensure that `gitlab.rb` on that node contains the configuration line `gitlab_rails['geo_node_name'] = 'node_name'`, where `node_name` is the same as the `geo_name_name` on the application node. @@ -288,7 +288,7 @@ For more information on how to replicate the Container Registry, see [Docker Reg ### Geo secondary proxy -For more information on using Geo proxying on secondary nodes, see [Geo proxying for secondary sites](secondary_proxy/index.md). +For more information on using Geo proxying on secondary sites, see [Geo proxying for secondary sites](secondary_proxy/index.md). ### Security Review diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 16b4848e6d3..adf89c24a4e 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -129,7 +129,7 @@ keys must be manually replicated to the **secondary** site. ```shell chown root:root /etc/ssh/ssh_host_*_key* - chmod 0600 /etc/ssh/ssh_host_*_key* + chmod 0600 /etc/ssh/ssh_host_*_key ``` 1. To verify key fingerprint matches, execute the following command on both primary and secondary nodes on each site: @@ -241,15 +241,60 @@ that the **secondary** site can act on those notifications immediately. Be sure the _secondary_ site is running and accessible. You can sign in to the _secondary_ site with the same credentials as were used with the _primary_ site. -### Step 4. (Optional) Configuring the **secondary** site to trust the **primary** site +### Step 4. (Optional) Using custom certificates -You can safely skip this step if your **primary** site uses a CA-issued HTTPS certificate. +You can safely skip this step if: -If your **primary** site is using a self-signed certificate for *HTTPS* support, you -need to add that certificate to the **secondary** site's trust store. Retrieve the -certificate from the **primary** site and follow -[these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) -on the **secondary** site. +- Your **primary** site uses a public CA-issued HTTPS certificate. +- Your **primary** site only connects to external services with CA-issued (not self-signed) HTTPS certificates. + +#### Custom or self-signed certificate for inbound connections + +If your GitLab Geo **primary** site uses a custom or [self-signed certificate to secure inbound HTTPS connections](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates), this certificate can either be single-domain certificate or multi-domain. + +Install the correct certificate based on your certificate type: + +- **Multi-domain certificate** that includes both primary and secondary site domains: Install the certificate at `/etc/gitlab/ssl` on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. +- **Single-domain certificate** where the certificates are specific to each Geo site domain: Generate a valid certificate for your **secondary** site's domain and install it at `/etc/gitlab/ssl` per [these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. + +#### Connecting to external services that use customer certificates + +A copy of the self-signed certificate for the external service needs to be added to the trust store on all the **primary** site's nodes that require access to the service. + +For the **secondary** site to be able to access the same external services, these certificates *must* be added to the **secondary** site's trust store. + +If your **primary** site is using a [custom or self-signed certificate for inbound HTTPS connections](#custom-or-self-signed-certificate-for-inbound-connections), the **primary** site's certificate needs to be added to the **secondary** site's trust store: + +1. SSH into each **Rails, Sidekiq, and Gitaly node on your secondary** site and login as root: + + ```shell + sudo -i + ``` + +1. Copy the trusted certs from the **primary** site: + + If you can access one of the nodes on your **primary** site serving SSH traffic using the root user: + + ```shell + scp root@<primary_site_node_fqdn>:/etc/gitlab/trusted-certs/* /etc/gitlab/trusted-certs + ``` + + If you only have access through a user with sudo privileges: + + ```shell + # Run this from the node on your primary site: + sudo tar --transform 's/.*\///g' -zcvf ~/geo-trusted-certs.tar.gz /etc/gitlab/trusted-certs/* + + # Run this on each node on your secondary site: + scp <user_with_sudo>@<primary_site_node_fqdn>:geo-trusted-certs.tar.gz . + tar zxvf ~/geo-trusted-certs.tar.gz -C /etc/gitlab/trusted-certs + ``` + +1. Reconfigure each updated **Rails, Sidekiq, and Gitaly node in your secondary** site: + + ```shell + sudo gitlab-ctl reconfigure + ``` ### Step 5. Enable Git access over HTTP/HTTPS diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 87b1aa7fc44..7b800817461 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -119,7 +119,7 @@ NOTE: [NFS](../../nfs.md) can be used in place of Gitaly but is not recommended. -### Step 2: Configure Postgres streaming replication +### Step 2: Configure PostgreSQL streaming replication Follow the [Geo database replication instructions](../setup/database.md). @@ -261,7 +261,7 @@ nodes connect to the databases. NOTE: Make sure that current node's IP is listed in `postgresql['md5_auth_cidr_addresses']` setting of the read-replica database to -allow Rails on this node to connect to Postgres. +allow Rails on this node to connect to PostgreSQL. After making these changes [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) so the changes take effect. diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 871d6041066..5a29c5a3c54 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -419,6 +419,21 @@ sudo gitlab-ctl reconfigure To help us resolve this problem, consider commenting on [the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/4489). +### Message: `FATAL: could not connect to the primary server: server certificate for "PostgreSQL" does not match host name` + +This happens because the PostgreSQL certificate that the Omnibus GitLab package automatically creates contains +the Common Name `PostgreSQL`, but the replication is connecting to a different host and GitLab attempts to use +the `verify-full` SSL mode by default. + +In order to fix this, you can either: + +- Use the `--sslmode=verify-ca` argument with the `replicate-geo-database` command. +- For an already replicated database, change `sslmode=verify-full` to `sslmode=verify-ca` + in `/var/opt/gitlab/postgresql/data/gitlab-geo.conf` and run `gitlab-ctl restart postgresql`. +- [Configure SSL for PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#configuring-ssl) + with a custom certificate (including the host name that's used to connect to the database in the CN or SAN) + instead of using the automatically generated certificate. + ### Message: `LOG: invalid CIDR mask in address` This happens on wrongly-formatted addresses in `postgresql['md5_auth_cidr_addresses']`. @@ -637,9 +652,9 @@ to start again from scratch, there are a few steps that can help you: 1. Reset the Tracking Database. ```shell - gitlab-rake geo:db:drop # on a secondary app node - gitlab-ctl reconfigure # on the tracking database node - gitlab-rake geo:db:setup # on a secondary app node + gitlab-rake db:drop:geo # on a secondary app node + gitlab-ctl reconfigure # on the tracking database node + gitlab-rake db:migrate:geo # on a secondary app node ``` 1. Restart previously stopped services. @@ -977,7 +992,7 @@ On the **primary** node: 1. On the left sidebar, select **Geo > Nodes**. 1. Find the affected **secondary** site and select **Edit**. 1. Ensure the **URL** field matches the value found in `/etc/gitlab/gitlab.rb` - in `external_url "https://gitlab.example.com"` on the frontend server(s) of + in `external_url "https://gitlab.example.com"` on the frontend servers of the **secondary** node. ## Fixing common errors @@ -1042,7 +1057,7 @@ Make sure you follow the [Geo database replication](../setup/database.md) instru If you are using Omnibus GitLab installation, something might have failed during upgrade. You can: - Run `sudo gitlab-ctl reconfigure`. -- Manually trigger the database migration by running: `sudo gitlab-rake geo:db:migrate` as root on the **secondary** node. +- Manually trigger the database migration by running: `sudo gitlab-rake db:migrate:geo` as root on the **secondary** node. ### GitLab indicates that more than 100% of repositories were synced @@ -1101,12 +1116,70 @@ This is due to [Pages data not being managed by Geo](datatypes.md#limitations-on Find advice to resolve those error messages in the [Pages administration documentation](../../../administration/pages/index.md#404-error-after-promoting-a-geo-secondary-to-a-primary-node). +### Primary site returns 500 error when accessing `/admin/geo/replication/projects` + +Navigating to **Admin > Geo > Replication** (or `/admin/geo/replication/projects`) on a primary Geo site, shows a 500 error, while that same link on the secondary works fine. The primary's `production.log` has a similar entry to the following: + +```plaintext +Geo::TrackingBase::SecondaryNotConfigured: Geo secondary database is not configured + from ee/app/models/geo/tracking_base.rb:26:in `connection' + [..] + from ee/app/views/admin/geo/projects/_all.html.haml:1 +``` + +On a Geo primary site this error can be ignored. + +This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). + ## Fixing client errors -### Authorization errors from LFS HTTP(s) client requests +### Authorization errors from LFS HTTP(S) client requests You may have problems if you're running a version of [Git LFS](https://git-lfs.github.com/) before 2.4.2. As noted in [this authentication issue](https://github.com/git-lfs/git-lfs/issues/3025), requests redirected from the secondary to the primary node do not properly send the Authorization header. This may result in either an infinite `Authorization <-> Redirect` loop, or Authorization error messages. + +## Recovering from a partial failover + +The partial failover to a secondary Geo *site* may be the result of a temporary/transient issue. Therefore, first attempt to run the promote command again. + +1. SSH into every Sidekiq, PostgresSQL, Gitaly, and Rails node in the **secondary** site and run one of the following commands: + + - To promote the secondary node to primary: + + ```shell + sudo gitlab-ctl geo promote + ``` + + - To promote the secondary node to primary **without any further confirmation**: + + ```shell + sudo gitlab-ctl geo promote --force + ``` + +1. Verify you can connect to the newly-promoted **primary** site using the URL used previously for the **secondary** site. +1. If **successful**, the **secondary** site is now promoted to the **primary** site. + +If the above steps are **not successful**, proceed through the next steps: + +1. SSH to every Sidekiq, PostgresSQL, Gitaly and Rails node in the **secondary** site and perform the following operations: + + - Create a `/etc/gitlab/gitlab-cluster.json` file with the following content: + + ```shell + { + "primary": true, + "secondary": false + } + ``` + + - Reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Verify you can connect to the newly-promoted **primary** site using the URL used previously for the **secondary** site. +1. If successful, the **secondary** site is now promoted to the **primary** site. diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_updates.md index b0797445890..6b617a21be8 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_updates.md @@ -12,9 +12,9 @@ for updating Geo sites. ## Updating to 14.9 -**DO NOT** update to GitLab 14.9.0. +**DO NOT** update to GitLab 14.9.0. Instead, use 14.9.1 or later. -We've discovered an issue with Geo's CI verification feature that may [cause job traces to be lost](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6664). This issue will be fixed in the next patch release. +We've discovered an issue with Geo's CI verification feature that may [cause job traces to be lost](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6664). This issue was fixed in [the GitLab 14.9.1 patch release](https://about.gitlab.com/releases/2022/03/23/gitlab-14-9-1-released/). If you have already updated to GitLab 14.9.0, you can disable the feature causing the issue by [disabling the `geo_job_artifact_replication` feature flag](../../feature_flags.md#how-to-enable-and-disable-features-behind-flags). diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index 9c917be123e..52f4adc4e03 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -300,6 +300,24 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o need it when setting up the **secondary** node! The certificate is not sensitive data. + However, this certificate is created with a generic `PostgreSQL` Common Name. For this, + you must use the `verify-ca` mode when replicating the database, otherwise, + the hostname mismatch will cause errors. + +1. Optional. Generate your own SSL certificate and manually + [configure SSL for PostgreSQL](https://docs.gitlab.com/omnibus/settings/database.html#configuring-ssl), + instead of using the generated certificate. + + You will need at least the SSL certificate and key, and set the `postgresql['ssl_cert_file']` and + `postgresql['ssl_key_file']` values to their full paths, as per the Database SSL docs. + + This allows you to use the `verify-full` SSL mode when replicating the database + and get the extra benefit of verifying the full hostname in the CN. + + You can use this certificate (that you have also set in `postgresql['ssl_cert_file']`) instead + of the certificate from the point above going forward. This will allow you to use `verify-full` + without replication errors if the CN matches. + #### Step 2. Configure the **secondary** server 1. SSH into your GitLab **secondary** server and login as root: @@ -367,7 +385,13 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o -h <primary_node_ip> ``` - When prompted enter the password you set in the first step for the + NOTE: + If you are using manually generated certificates and plan on using + `sslmode=verify-full` to benefit of the full hostname verification, + make sure to replace `verify-ca` to `verify-full` when + running the command. + + When prompted enter the _plaintext_ password you set in the first step for the `gitlab_replicator` user. If all worked correctly, you should see the list of **primary** node's databases. @@ -455,6 +479,7 @@ data before running `pg_basebackup`. gitlab-ctl replicate-geo-database \ --slot-name=<secondary_node_name> \ --host=<primary_node_ip> + --sslmode=verify-ca ``` NOTE: @@ -463,6 +488,13 @@ data before running `pg_basebackup`. When prompted, enter the _plaintext_ password you set up for the `gitlab_replicator` user in the first step. + NOTE: + If you have generated custom PostgreSQL certificates, you will want to use + `--sslmode=verify-full` (or omit the `sslmode` line entirely), to benefit from the extra + validation of the full host name in the certificate CN / SAN for additional security. + Otherwise, using the automatically created certificate with `verify-full` will fail, + as it has a generic `PostgreSQL` CN which will not match the `--host` value in this command. + This command also takes a number of additional options. You can use `--help` to list them all, but here are a couple of tips: @@ -1061,7 +1093,7 @@ For each node running the `gitlab-rails`, `sidekiq`, and `geo-logcursor` service 1. Run the tracking database migrations: ```shell - gitlab-rake geo:db:migrate + gitlab-rake db:migrate:geo ``` ### Migrating a single tracking database node to Patroni diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index cd77647e7dc..58fd6b28797 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -245,11 +245,11 @@ the tracking database on port 5432. 1. The reconfigure should automatically create the database. If needed, you can perform this task manually. This task (whether run by itself or during reconfigure) requires the database user to be a superuser. ```shell - gitlab-rake geo:db:create + gitlab-rake db:create:geo ``` 1. The reconfigure should automatically migrate the database. You can migrate the database manually if needed, for example if `geo_secondary['auto_migrate'] = false`: ```shell - gitlab-rake geo:db:migrate + gitlab-rake db:migrate:geo ``` diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index 29156d9b3e1..3612487f456 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -111,4 +111,4 @@ URL to use SSH. ### Observe Git protocol version of connections For information on observing the Git protocol versions are being used in a production environment, -see the [relevant documentation](gitaly/index.md#useful-queries). +see the [relevant documentation](gitaly/monitoring.md#useful-queries). diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 0fb285c50d6..6888a2abe9a 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Configure Gitaly **(FREE SELF)** @@ -217,7 +216,12 @@ disable enforcement. For more information, see the documentation on configuring 1. Edit `/etc/gitlab/gitlab.rb`: - <!-- Updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> ```ruby # Avoid running unnecessary services on the Gitaly server @@ -229,10 +233,11 @@ disable enforcement. For more information, see the documentation on configuring gitlab_workhorse['enable'] = false grafana['enable'] = false gitlab_exporter['enable'] = false + gitlab_kas['enable'] = false # If you run a separate monitoring node you can disable these services - alertmanager['enable'] = false prometheus['enable'] = false + alertmanager['enable'] = false # If you don't run a separate monitoring node you can # enable Prometheus access & disable these extra services. @@ -252,14 +257,14 @@ disable enforcement. For more information, see the documentation on configuring # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from Gitaly client to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Authentication token to ensure only authorized servers can communicate with - # Gitaly server - gitaly['auth_token'] = 'AUTH_TOKEN' - # Make Gitaly accept connections on all network interfaces. You must use # firewalls to restrict access to this address/port. # Comment out following line if you only want to support TLS connections gitaly['listen_addr'] = "0.0.0.0:8075" + + # Authentication token to ensure only authorized servers can communicate with + # Gitaly server + gitaly['auth_token'] = 'AUTH_TOKEN' ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective Gitaly server: @@ -710,7 +715,7 @@ To configure Gitaly with TLS: ### Observe type of Gitaly connections For information on observing the type of Gitaly connections being served, see the -[relevant documentation](index.md#useful-queries). +[relevant documentation](monitoring.md#useful-queries). ## `gitaly-ruby` @@ -773,8 +778,8 @@ settings: Clone traffic can put a large strain on your Gitaly service. The bulk of the work gets done in the either of the following RPCs: -- `SSHUploadPack` (for Git SSH). -- `PostUploadPack` (for Git HTTP). +- `SSHUploadPackWithSidechannel` (for Git SSH). +- `PostUploadPackWithSidechannel` (for Git HTTP). To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in Gitaly's configuration file. For example: @@ -784,26 +789,103 @@ Gitaly's configuration file. For example: gitaly['concurrency'] = [ { - 'rpc' => "/gitaly.SmartHTTPService/PostUploadPack", - 'max_per_repo' => 20 + 'rpc' => "/gitaly.SmartHTTPService/PostUploadPackWithSidechanel", + 'max_per_repo' => 20, + 'max_queue_time' => "1s", + 'max_queue_size' => 10 }, { - 'rpc' => "/gitaly.SSHService/SSHUploadPack", + 'rpc' => "/gitaly.SSHService/SSHUploadPackWithSidechannel", 'max_per_repo' => 20 + 'max_queue_time' => "1s", + 'max_queue_size' => 10 } ] ``` +- `rpc` is the name of the RPC to set a concurrency limit for per repository. +- `max_per_repo` is the maximum number of in-flight RPC calls for the given RPC per repository. +- `max_queue_time` is the maximum amount of time a request can wait in the concurrency queue to + be picked up by Gitaly. +- `max_queue_size` is the maximum size the concurrency queue can grow to before requests are rejected by + Gitaly. + This limits the number of in-flight RPC calls for the given RPCs. The limit is applied per repository. In the example above: -- Each repository served by the Gitaly server can have at most 20 simultaneous `PostUploadPack` RPC - calls in flight, and the same for `SSHUploadPack`. +- Each repository served by the Gitaly server can have at most 20 simultaneous `PostUploadPackWithSidechannel` and + `SSHUploadPackWithSidechannel` RPC calls in flight. - If another request comes in for a repository that has used up its 20 slots, that request gets queued. +- If a request waits in the queue for more than 1 second, it is rejected with an error. +- If the queue grows beyond 10, subsequent requests are rejected with an error. You can observe the behavior of this queue using the Gitaly logs and Prometheus. For more -information, see the [relevant documentation](index.md#monitor-gitaly). +information, see the [relevant documentation](monitoring.md#monitor-gitaly-concurrency-limiting). + +## Control groups + +> - Introduced in GitLab 13.10. + +Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations, +especially for large repositories. + +Control groups (cgroups) in Linux allow limits to be imposed on how much memory and CPU can be consumed. +See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information. +cgroups can be useful for protecting the system against resource exhaustion because of overconsumption of memory and CPU. + +Gitaly has built-in cgroups control. When configured, Gitaly assigns Git +processes to a cgroup based on the repository the Git command is operating in. +Each cgroup has a memory and CPU limit. When a cgroup reaches its: + +- Memory limit, the kernel looks through the processes for a candidate to kill. +- CPU limit, processes are not killed, but the processes are prevented from consuming more CPU than allowed. + +The main reason to configure cgroups for your GitLab installation is that it +protects against system resource starvation due to a few large repositories or +bad actors. + +Some Git operations are expensive by nature. `git clone`, for instance, +spawns a `git-upload-pack` process on the server that can consume a lot of memory +for large repositories. For example, a client that keeps on cloning a +large repository over and over again. This situation could potentially use up all of the +memory on a server, causing other operations to fail for other users. + +There are many ways someone can create a repository that can consume large amounts of memory when cloned or downloaded. +Using cgroups allows the kernel to kill these operations before they hog up all system resources. + +### Configure cgroups in Gitaly + +To configure cgroups in Gitaly, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. For +example: + +```ruby +# in /etc/gitlab/gitlab.rb +gitaly['cgroups_count'] = 1000 +gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" +gitaly['cgroups_hierarchy_root'] = "gitaly" +gitaly['cgroups_memory_limit'] = 32212254720 +gitaly['cgroups_memory_enabled'] = true +gitaly['cgroups_cpu_shares'] = 1024 +gitaly['cgroups_cpu_enabled'] = true + +``` + +- `cgroups_count` is the number of cgroups created. Each time a new + command is spawned, Gitaly assigns it to one of these cgroups based + on the command line arguments of the command. A circular hashing algorithm assigns + commands to these cgroups. +- `cgroups_mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. +- `cgroups_hierarchy_root` is the parent cgroup under which Gitaly creates groups, and + is expected to be owned by the user and group Gitaly runs as. Omnibus GitLab + creates the set of directories `mountpoint/<cpu|memory>/hierarchy_root` + when Gitaly starts. +- `cgroups_memory_enabled` enables or disables the memory limit on cgroups. +- `cgroups_memory_bytes` is the total memory limit each cgroup imposes on the processes added to it. +- `cgroups_cpu_enabled` enables or disables the CPU limit on cgroups. +- `cgroups_cpu_shares` is the CPU limit each cgroup imposes on the processes added to it. The maximum is 1024 shares, + which represents 100% of CPU. + which represents 100% of CPU. ## Background Repository Optimization @@ -858,7 +940,7 @@ server" and "Gitaly client" refers to the same machine. ### Verify authentication monitoring Before rotating a Gitaly authentication token, verify that you can -[monitor the authentication behavior](index.md#useful-queries) of your GitLab installation using +[monitor the authentication behavior](monitoring.md#useful-queries) of your GitLab installation using Prometheus. You can then continue the rest of the procedure. @@ -1068,7 +1150,7 @@ closed it. ### Observe the cache -The cache can be observed [using metrics](index.md#monitor-gitaly) and in the following logged +The cache can be observed [using metrics](monitoring.md#pack-objects-cache) and in the following logged information: |Message|Fields|Description| diff --git a/doc/administration/gitaly/faq.md b/doc/administration/gitaly/faq.md index b0a88e8adc9..a5c2c7d1469 100644 --- a/doc/administration/gitaly/faq.md +++ b/doc/administration/gitaly/faq.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Frequently asked questions **(FREE SELF)** diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 8e4464bba43..f43f9f5d6c1 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Gitaly and Gitaly Cluster **(FREE SELF)** @@ -57,7 +56,7 @@ If you have: - Not yet migrated to Gitaly Cluster and want to continue using NFS, remain on the service you are using. NFS is supported in 14.x releases but is [deprecated](../../update/deprecations.md#nfs-for-git-repository-storage). -Support for storing Git repository data on NFS will end for all versions of GitLab with the release of 15.0. + Support for storing Git repository data on NFS is scheduled to end for all versions of GitLab on November 22, 2022. - Not yet migrated to Gitaly Cluster but want to migrate away from NFS, you have two options: - A sharded Gitaly instance. - Gitaly Cluster. @@ -72,7 +71,7 @@ the current status of these issues, please refer to the referenced issues and ep | Issue | Summary | How to avoid | |:--------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------| | Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. Work is in-progress to update Gitaly Cluster to [identify repositories with a unique and persistent identifier](https://gitlab.com/gitlab-org/gitaly/-/issues/3485), which is expected to resolve the issue. | No known solution at this time. | -| Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform normal operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting live upgrade assistance](https://about.gitlab.com/support/scheduling-live-upgrade-assistance.html) so your upgrade plan can be reviewed by support. | +| Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform normal operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting live upgrade assistance](https://about.gitlab.com/support/scheduling-upgrade-assistance.html) so your upgrade plan can be reviewed by support. | | Restoring a Gitaly Cluster node from a snapshot in a running cluster | Because the Gitaly Cluster runs with consistent state, introducing a single node that is behind will result in the cluster not being able to reconcile the nodes data and other nodes data | Don't restore a single Gitaly Cluster node from a backup snapshot. If you must restore from backup, it's best to snapshot all Gitaly Cluster nodes at the same time and take a database dump of the Praefect database. | ### Snapshot backup and recovery limitations @@ -233,9 +232,142 @@ variable replication factor is tracked in [this issue](https://gitlab.com/groups As with normal Gitaly storages, virtual storages can be sharded. +### Storage layout + +WARNING: +The storage layout is an internal detail of Gitaly Cluster and is not guaranteed to remain stable between releases. +The information here is only for informational purposes and to help with debugging. Performing changes in the +repositories directly on the disk is not supported and may lead to breakage or the changes being overwritten. + +Gitaly Cluster's virtual storages provide an abstraction that looks like a single storage but actually consists of +multiple physical storages. Gitaly Cluster has to replicate each operation to each physical storage. Operations +may succeed on some of the physical storages but fail on others. + +Partially applied operations can cause problems with other operations and leave the system in a state it can't recover from. +To avoid these types of problems, each operation should either fully apply or not apply at all. This property of operations is called +[atomicity](https://en.wikipedia.org/wiki/Atomicity_(database_systems)). + +GitLab controls the storage layout on the repository storages. GitLab instructs the repository storage where to create, +delete, and move repositories. These operations create atomicity issues when they are being applied to multiple physical storages. +For example: + +- GitLab deletes a repository while one of its replicas is unavailable. +- GitLab later recreates the repository. + +As a result, the stale replica that was unavailable at the time of deletion may cause conflicts and prevent +recreation of the repository. + +These atomicity issues have caused multiple problems in the past with: + +- Geo syncing to a secondary site with Gitaly Cluster. +- Backup restoration. +- Repository moves between repository storages. + +Gitaly Cluster provides atomicity for these operations by storing repositories on the disk in a special layout that prevents +conflicts that could occur due to partially applied operations. + +#### Client-generated replica paths + +Repositories are stored in the storages at the relative path determined by the [Gitaly client](#gitaly-architecture). These paths can be +identified by them not beginning with the `@cluster` prefix. The relative paths +follow the [hashed storage](../repository_storage_types.md#hashed-storage) schema. + +#### Praefect-generated replica paths (GitLab 15.0 and later) + +> Introduced in GitLab 15.0 behind [a feature flag](https://gitlab.com/gitlab-org/gitaly/-/issues/4218) named `gitaly_praefect_generated_replica_paths`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) +named `gitaly_praefect_generated_replica_paths`. On GitLab.com, this feature is available but can be configured by GitLab.com administrators only. The feature is not ready for production use. + +When Gitaly Cluster creates a repository, it assigns the repository a unique and permanent ID called the _repository ID_. The repository ID is +internal to Gitaly Cluster and doesn't relate to any IDs elsewhere in GitLab. If a repository is removed from Gitaly Cluster and later moved +back, the repository is assigned a new repository ID and is a different repository from Gitaly Cluster's perspective. The sequence of repository IDs +always increases, but there may be gaps in the sequence. + +The repository ID is used to derive a unique storage path called _replica path_ for each repository on the cluster. The replicas of +a repository are all stored at the same replica path on the storages. The replica path is distinct from the _relative path_: + +- The relative path is a name the Gitaly client uses to identify a repository, together with its virtual storage, that is unique to them. +- The replica path is the actual physical path in the physical storages. + +Praefect translates the repositories in the RPCs from the virtual `(virtual storage, relative path)` identifier into physical repository +`(storage, replica_path)` identifier when handling the client requests. + +The format of the replica path for: + +- Object pools is `@cluster/pools/<xx>/<xx>/<repository ID>`. Object pools are stored in a different directory than other repositories. + They must be identifiable by Gitaly to avoid pruning them as part of housekeeping. Pruning object pools can cause data loss in the linked + repositories. +- Other repositories is `@cluster/repositories/<xx>/<xx>/<repository ID>` + +For example, `@cluster/repositories/6f/96/54771`. + +The last component of the replica path, `54771`, is the repository ID. This can be used to identify the repository on the disk. + +`<xx>/<xx>` are the first four hex digits of the SHA256 hash of the string representation of the repository ID. This is used to balance +the repositories evenly into subdirectories to avoid overly large directories that might cause problems on some file +systems. In this case, `54771` hashes to `6f960ab01689464e768366d3315b3d3b2c28f38761a58a70110554eb04d582f7` so the +first four digits are `6f` and `96`. + +#### Identify repositories on disk + +Use the [`praefect metadata`](troubleshooting.md#view-repository-metadata) subcommand to: + +- Retrieve a repository's virtual storage and relative path from the metadata store. After you have the hashed storage path, you can use the Rails + console to retrieve the project path. +- Find where a repository is stored in the cluster with either: + - The virtual storage and relative path. + - The repository ID. + +The repository on disk also contains the project path in the Git configuration file. The configuration file can be used to determine +the project's location even if the repository's metadata has been deleted. Follow the +[instructions in hashed storage's documentation](../repository_storage_types.md#from-hashed-path-to-project-name). + +#### Atomicity of operations + +Gitaly Cluster uses the PostgreSQL metadata store with the storage layout to ensure atomicity of repository creation, +deletion, and move operations. The disk operations can't be atomically applied across multiple storages. However, PostgreSQL guarantees +the atomicity of the metadata operations. Gitaly Cluster models the operations in a manner that the failing operations always leave +the metadata consistent. The disks may contain stale state even after successful operations. This is expected and the leftover state +won't intefere with future operations but may use up disk space unnecessarily until a clean up is performed. + +There is on-going work on a [background crawler](https://gitlab.com/gitlab-org/gitaly/-/issues/3719) that cleans up the leftover +repositories from the storages. + +##### Repository creations + +When creating repositories, Praefect: + +1. Reserves a repository ID from PostgreSQL. This is atomic and no two creations receive the same ID. +1. Creates replicas on the Gitaly storages in the replica path derived from the repository ID. +1. Creates metadata records after the repository is successfully created on disk. + +Even if two concurrent operations create the same repository, they'd be stored in different directories on the storages and not +conflict. The first to complete creates the metadata record and the other operation fails with an "already exists" error. +The failing creation leaves leftover repositories on the storages. There is on-going work on a +[background crawler](https://gitlab.com/gitlab-org/gitaly/-/issues/3719) that clean up the leftover repositories from the storages. + +The repository IDs are generated from the `repositories_repository_id_seq` in PostgreSQL. In the above example, the failing operation took +one repository ID without successfully creating a repository with it. Failed repository creations are expected lead to gaps in the repository IDs. + +##### Repository deletions + +A repository is deleted by removing its metadata record. The repository ceases to logically exist as soon as the metadata record is deleted. +PostgreSQL guarantees the atomicity of the removal and a concurrent delete fails with a "not found" error. After successfully deleting +the metadata record, Praefect attempts to remove the replicas from the storages. This may fail and leave leftover state in the storages. +The leftover state is eventually cleaned up. + +##### Repository moves + +Unlike Gitaly, Gitaly Cluster doesn't move the repositories in the storages but only virtually moves the repository by updating the +relative path of the repository in the metadata store. + ### Moving beyond NFS -Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be unavailable starting GitLab 15.0. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for more details. +Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be unavailable starting +November 22, 2022. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) +for more details. [Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) is not well suited to Git workloads which are CPU and IOPS sensitive. @@ -316,7 +448,7 @@ The primary node is chosen to serve the request if: - There are no up to date nodes. - Any other error occurs during node selection. -You can [monitor distribution of reads](#monitor-gitaly-cluster) using Prometheus. +You can [monitor distribution of reads](monitoring.md#monitor-gitaly-cluster) using Prometheus. #### Strong consistency @@ -344,7 +476,7 @@ Strong consistency: - The [13.12 documentation](https://docs.gitlab.com/13.12/ee/administration/gitaly/praefect.html#strong-consistency). - Is unavailable in GitLab 13.0 and earlier. -For more information on monitoring strong consistency, see the Gitaly Cluster [Prometheus metrics documentation](#monitor-gitaly-cluster). +For more information on monitoring strong consistency, see the Gitaly Cluster [Prometheus metrics documentation](monitoring.md#monitor-gitaly-cluster). #### Replication factor @@ -392,167 +524,6 @@ off Gitaly Cluster to a sharded Gitaly instance: 1. [Move the repositories](../operations/moving_repositories.md#move-repositories) to the newly created storage. You can move them by shard or by group, which gives you the opportunity to spread them over multiple Gitaly servers. -## Monitor Gitaly and Gitaly Cluster - -You can use the available logs and [Prometheus metrics](../monitoring/prometheus/index.md) to -monitor Gitaly and Gitaly Cluster (Praefect). - -Metric definitions are available: - -- Directly from Prometheus `/metrics` endpoint configured for Gitaly. -- Using [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) on a - Grafana instance configured against Prometheus. - -### Monitor Gitaly - -You can observe the behavior of [queued requests](configure_gitaly.md#limit-rpc-concurrency) using -the Gitaly logs and Prometheus: - -- In the [Gitaly logs](../logs.md#gitaly-logs), look for the string (or structured log field) - `acquire_ms`. Messages that have this field are reporting about the concurrency limiter. -- In Prometheus, look for the following metrics: - - `gitaly_rate_limiting_in_progress`. - - `gitaly_rate_limiting_queued`. - - `gitaly_rate_limiting_seconds`. - - Although the name of the Prometheus metric contains `rate_limiting`, it's a concurrency limiter, - not a rate limiter. If a Gitaly client makes 1,000 requests in a row very quickly, concurrency - doesn't exceed 1, and the concurrency limiter has no effect. - -The following [pack-objects cache](configure_gitaly.md#pack-objects-cache) metrics are available: - -- `gitaly_pack_objects_cache_enabled`, a gauge set to `1` when the cache is enabled. Available - labels: `dir` and `max_age`. -- `gitaly_pack_objects_cache_lookups_total`, a counter for cache lookups. Available label: `result`. -- `gitaly_pack_objects_generated_bytes_total`, a counter for the number of bytes written into the - cache. -- `gitaly_pack_objects_served_bytes_total`, a counter for the number of bytes read from the cache. -- `gitaly_streamcache_filestore_disk_usage_bytes`, a gauge for the total size of cache files. - Available label: `dir`. -- `gitaly_streamcache_index_entries`, a gauge for the number of entries in the cache. Available - label: `dir`. - -Some of these metrics start with `gitaly_streamcache` because they are generated by the -`streamcache` internal library package in Gitaly. - -Example: - -```plaintext -gitaly_pack_objects_cache_enabled{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache",max_age="300"} 1 -gitaly_pack_objects_cache_lookups_total{result="hit"} 2 -gitaly_pack_objects_cache_lookups_total{result="miss"} 1 -gitaly_pack_objects_generated_bytes_total 2.618649e+07 -gitaly_pack_objects_served_bytes_total 7.855947e+07 -gitaly_streamcache_filestore_disk_usage_bytes{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 2.6200152e+07 -gitaly_streamcache_filestore_removed_total{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 -gitaly_streamcache_index_entries{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 -``` - -#### Useful queries - -The following are useful queries for monitoring Gitaly: - -- Use the following Prometheus query to observe the - [type of connections](configure_gitaly.md#enable-tls-support) Gitaly is serving a production - environment: - - ```prometheus - sum(rate(gitaly_connections_total[5m])) by (type) - ``` - -- Use the following Prometheus query to monitor the - [authentication behavior](configure_gitaly.md#observe-type-of-gitaly-connections) of your GitLab - installation: - - ```prometheus - sum(rate(gitaly_authentications_total[5m])) by (enforced, status) - ``` - - In a system where authentication is configured correctly and where you have live traffic, you - see something like this: - - ```prometheus - {enforced="true",status="ok"} 4424.985419441742 - ``` - - There may also be other numbers with rate 0, but you only have to take note of the non-zero numbers. - - The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero - numbers, something is wrong in your configuration. - - The `status="ok"` number reflects your current request rate. In the example above, Gitaly is - handling about 4000 requests per second. - -- Use the following Prometheus query to observe the [Git protocol versions](../git_protocol.md) - being used in a production environment: - - ```prometheus - sum(rate(gitaly_git_protocol_requests_total[1m])) by (grpc_method,git_protocol,grpc_service) - ``` - -### Monitor Gitaly Cluster - -To monitor Gitaly Cluster (Praefect), you can use these Prometheus metrics. There are two separate metrics -endpoints from which metrics can be scraped: - -- The default `/metrics` endpoint. -- `/db_metrics`, which contains metrics that require database queries. - -#### Default Prometheus `/metrics` endpoint - -The following metrics are available from the `/metrics` endpoint: - -- `gitaly_praefect_read_distribution`, a counter to track [distribution of reads](#distributed-reads). - It has two labels: - - - `virtual_storage`. - - `storage`. - - They reflect configuration defined for this instance of Praefect. - -- `gitaly_praefect_replication_latency_bucket`, a histogram measuring the amount of time it takes - for replication to complete after the replication job starts. Available in GitLab 12.10 and later. -- `gitaly_praefect_replication_delay_bucket`, a histogram measuring how much time passes between - when the replication job is created and when it starts. Available in GitLab 12.10 and later. -- `gitaly_praefect_node_latency_bucket`, a histogram measuring the latency in Gitaly returning - health check information to Praefect. This indicates Praefect connection saturation. Available in - GitLab 12.10 and later. - -To monitor [strong consistency](#strong-consistency), you can use the following Prometheus metrics: - -- `gitaly_praefect_transactions_total`, the number of transactions created and voted on. -- `gitaly_praefect_subtransactions_per_transaction_total`, the number of times nodes cast a vote for - a single transaction. This can happen multiple times if multiple references are getting updated in - a single transaction. -- `gitaly_praefect_voters_per_transaction_total`: the number of Gitaly nodes taking part in a - transaction. -- `gitaly_praefect_transactions_delay_seconds`, the server-side delay introduced by waiting for the - transaction to be committed. -- `gitaly_hook_transaction_voting_delay_seconds`, the client-side delay introduced by waiting for - the transaction to be committed. - -To monitor the number of repositories that have no healthy, up-to-date replicas: - -- `gitaly_praefect_unavailable_repositories` - -You can also monitor the [Praefect logs](../logs.md#praefect-logs). - -#### Database metrics `/db_metrics` endpoint - -> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/3286) in GitLab 14.5. - -The following metrics are available from the `/db_metrics` endpoint: - -- `gitaly_praefect_unavailable_repositories`, the number of repositories that have no healthy, up to date replicas. -- `gitaly_praefect_read_only_repositories`, the number of repositories in read-only mode in a virtual storage. - This metric is available for backwards compatibility reasons. `gitaly_praefect_unavailable_repositories` is more - accurate. -- `gitaly_praefect_replication_queue_depth`, the number of jobs in the replication queue. - -## Recover from failure - -Gitaly Cluster can [recover from certain types of failure](recovery.md). - ## Do not bypass Gitaly GitLab doesn't advise directly accessing Gitaly repositories stored on disk with a Git client, @@ -660,4 +631,4 @@ The second facet presents the only real solution. For this, we developed ## NFS deprecation notice Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be -unavailable from GitLab 15.0. For further information, please see our [NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation) documentation. +unavailable beginning November 22, 2022. For further information, please see our [NFS Deprecation](../nfs.md#gitaly-and-nfs-deprecation) documentation. diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md new file mode 100644 index 00000000000..17f94f912ee --- /dev/null +++ b/doc/administration/gitaly/monitoring.md @@ -0,0 +1,202 @@ +--- +stage: Create +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Monitoring Gitaly and Gitaly Cluster + +You can use the available logs and [Prometheus metrics](../monitoring/prometheus/index.md) to +monitor Gitaly and Gitaly Cluster (Praefect). + +Metric definitions are available: + +- Directly from Prometheus `/metrics` endpoint configured for Gitaly. +- Using [Grafana Explore](https://grafana.com/docs/grafana/latest/explore/) on a + Grafana instance configured against Prometheus. + +## Monitor Gitaly rate limiting + +Gitaly can be configured to limit requests based on: + +- Concurrency of requests. +- A rate limit. + +Monitor Gitaly request limiting with the `gitaly_requests_dropped_total` Prometheus metric. This metric provides a total count +of requests dropped due to request limiting. The `reason` label indicates why a request was dropped: + +- `rate`, due to rate limiting. +- `max_size`, because the concurrency queue size was reached. +- `max_time`, because the request exceeded the maximum queue wait time as configured in Gitaly. + +## Monitor Gitaly concurrency limiting + +You can observe specific behavior of [concurrency-queued requests](configure_gitaly.md#limit-rpc-concurrency) using +the Gitaly logs and Prometheus: + +- In the [Gitaly logs](../logs.md#gitaly-logs), look for the string (or structured log field) + `acquire_ms`. Messages that have this field are reporting about the concurrency limiter. +- In Prometheus, look for the following metrics: + - `gitaly_concurrency_limiting_in_progress` indicates how many concurrent requests are + being processed. + - `gitaly_concurrency_limiting_queued` indicates how many requests for an RPC for a given + repository are waiting due to the concurrency limit being reached. + - `gitaly_concurrency_limiting_acquiring_seconds` indicates how long a request has to + wait due to concurrency limits before being processed. + +## Monitor Gitaly cgroups + +You can observe the status of [control groups (cgroups)](configure_gitaly.md#control-groups) using Prometheus: + +- `gitaly_cgroups_memory_failed_total`, a gauge for the total number of times + the memory limit has been hit. This number resets each time a server is + restarted. +- `gitaly_cgroups_cpu_usage`, a gauge that measures CPU usage per cgroup. +- `gitaly_cgroup_procs_total`, a gauge that measures the total number of + processes Gitaly has spawned under the control of cgroups. + +## `pack-objects` cache + +The following [`pack-objects` cache](configure_gitaly.md#pack-objects-cache) metrics are available: + +- `gitaly_pack_objects_cache_enabled`, a gauge set to `1` when the cache is enabled. Available + labels: `dir` and `max_age`. +- `gitaly_pack_objects_cache_lookups_total`, a counter for cache lookups. Available label: `result`. +- `gitaly_pack_objects_generated_bytes_total`, a counter for the number of bytes written into the + cache. +- `gitaly_pack_objects_served_bytes_total`, a counter for the number of bytes read from the cache. +- `gitaly_streamcache_filestore_disk_usage_bytes`, a gauge for the total size of cache files. + Available label: `dir`. +- `gitaly_streamcache_index_entries`, a gauge for the number of entries in the cache. Available + label: `dir`. + +Some of these metrics start with `gitaly_streamcache` because they are generated by the +`streamcache` internal library package in Gitaly. + +Example: + +```plaintext +gitaly_pack_objects_cache_enabled{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache",max_age="300"} 1 +gitaly_pack_objects_cache_lookups_total{result="hit"} 2 +gitaly_pack_objects_cache_lookups_total{result="miss"} 1 +gitaly_pack_objects_generated_bytes_total 2.618649e+07 +gitaly_pack_objects_served_bytes_total 7.855947e+07 +gitaly_streamcache_filestore_disk_usage_bytes{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 2.6200152e+07 +gitaly_streamcache_filestore_removed_total{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 +gitaly_streamcache_index_entries{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 +``` + +## Useful queries + +The following are useful queries for monitoring Gitaly: + +- Use the following Prometheus query to observe the + [type of connections](configure_gitaly.md#enable-tls-support) Gitaly is serving a production + environment: + + ```prometheus + sum(rate(gitaly_connections_total[5m])) by (type) + ``` + +- Use the following Prometheus query to monitor the + [authentication behavior](configure_gitaly.md#observe-type-of-gitaly-connections) of your GitLab + installation: + + ```prometheus + sum(rate(gitaly_authentications_total[5m])) by (enforced, status) + ``` + + In a system where authentication is configured correctly and where you have live traffic, you + see something like this: + + ```prometheus + {enforced="true",status="ok"} 4424.985419441742 + ``` + + There may also be other numbers with rate 0, but you only have to take note of the non-zero numbers. + + The only non-zero number should have `enforced="true",status="ok"`. If you have other non-zero + numbers, something is wrong in your configuration. + + The `status="ok"` number reflects your current request rate. In the example above, Gitaly is + handling about 4000 requests per second. + +- Use the following Prometheus query to observe the [Git protocol versions](../git_protocol.md) + being used in a production environment: + + ```prometheus + sum(rate(gitaly_git_protocol_requests_total[1m])) by (grpc_method,git_protocol,grpc_service) + ``` + +## Monitor Gitaly Cluster + +To monitor Gitaly Cluster (Praefect), you can use these Prometheus metrics. There are two separate metrics +endpoints from which metrics can be scraped: + +- The default `/metrics` endpoint. +- `/db_metrics`, which contains metrics that require database queries. + +### Default Prometheus `/metrics` endpoint + +The following metrics are available from the `/metrics` endpoint: + +- `gitaly_praefect_read_distribution`, a counter to track [distribution of reads](index.md#distributed-reads). + It has two labels: + + - `virtual_storage`. + - `storage`. + + They reflect configuration defined for this instance of Praefect. + +- `gitaly_praefect_replication_latency_bucket`, a histogram measuring the amount of time it takes + for replication to complete after the replication job starts. Available in GitLab 12.10 and later. +- `gitaly_praefect_replication_delay_bucket`, a histogram measuring how much time passes between + when the replication job is created and when it starts. Available in GitLab 12.10 and later. +- `gitaly_praefect_node_latency_bucket`, a histogram measuring the latency in Gitaly returning + health check information to Praefect. This indicates Praefect connection saturation. Available in + GitLab 12.10 and later. + +To monitor [strong consistency](index.md#strong-consistency), you can use the following Prometheus metrics: + +- `gitaly_praefect_transactions_total`, the number of transactions created and voted on. +- `gitaly_praefect_subtransactions_per_transaction_total`, the number of times nodes cast a vote for + a single transaction. This can happen multiple times if multiple references are getting updated in + a single transaction. +- `gitaly_praefect_voters_per_transaction_total`: the number of Gitaly nodes taking part in a + transaction. +- `gitaly_praefect_transactions_delay_seconds`, the server-side delay introduced by waiting for the + transaction to be committed. +- `gitaly_hook_transaction_voting_delay_seconds`, the client-side delay introduced by waiting for + the transaction to be committed. + +To monitor the number of repositories that have no healthy, up-to-date replicas: + +- `gitaly_praefect_unavailable_repositories` + +To monitor [repository verification](praefect.md#repository-verification), use the following Prometheus metrics: + +- `gitaly_praefect_verification_queue_depth`, the total number of replicas pending verification. This + metric is scraped from the database and is only available when Prometheus is scraping the database metrics. +- `gitaly_praefect_verification_jobs_dequeued_total`, the number of verification jobs picked up by the + worker. +- `gitaly_praefect_verification_jobs_completed_total`, the number of verification jobs completed by the + worker. The `result` label indicates the end result of the jobs: + - `valid` indicates the expected replica existed on the storage. + - `invalid` indicates the replica expected to exist did not exist on the storage. + - `error` indicates the job failed and has to be retried. +- `gitaly_praefect_stale_verification_leases_released_total`, the number of stale verification leases + released. + +You can also monitor the [Praefect logs](../logs.md#praefect-logs). + +### Database metrics `/db_metrics` endpoint + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/3286) in GitLab 14.5. + +The following metrics are available from the `/db_metrics` endpoint: + +- `gitaly_praefect_unavailable_repositories`, the number of repositories that have no healthy, up to date replicas. +- `gitaly_praefect_read_only_repositories`, the number of repositories in read-only mode in a virtual storage. + This metric is available for backwards compatibility reasons. `gitaly_praefect_unavailable_repositories` is more + accurate. +- `gitaly_praefect_replication_queue_depth`, the number of jobs in the replication queue. diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 7dee12a6690..fb9f3e9c361 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Configure Gitaly Cluster **(FREE SELF)** @@ -326,7 +325,7 @@ pgbouncer['databases'] = { user: 'pgbouncer', password: PGBOUNCER_SQL_PASSWORD_HASH, pool_mode: 'transaction' - } + }, praefect_production_direct: { host: POSTGRESQL_HOST, # Use `pgbouncer` user to connect to database backend. @@ -338,6 +337,13 @@ pgbouncer['databases'] = { ... } + +# Allow the praefect user to connect to PgBouncer +pgbouncer['users'] = { + 'praefect': { + 'password': PRAEFECT_SQL_PASSWORD_HASH, + } +} ``` Both `praefect_production` and `praefect_production_direct` use the same database endpoint @@ -422,25 +428,33 @@ On the **Praefect** node: 1. Disable all other services by editing `/etc/gitlab/gitlab.rb`: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby - # Disable all other services on the Praefect node + # Avoid running unnecessary services on the Praefect server + gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false - alertmanager['enable'] = false - prometheus['enable'] = false - grafana['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false - gitaly['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + gitlab_kas['enable'] = false # Enable only the Praefect service praefect['enable'] = true - # Disable database migrations to prevent database connections during 'gitlab-ctl reconfigure' - gitlab_rails['auto_migrate'] = false + # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false + gitlab_rails['auto_migrate'] = false ``` 1. Configure **Praefect** to listen on network interfaces by editing @@ -953,7 +967,7 @@ application. This is done by updating the `git_data_dirs`. Particular attention should be shown to: - the storage name added to `git_data_dirs` in this section must match the - storage name under `praefect['virtual_storages']` on the Praefect node(s). This + storage name under `praefect['virtual_storages']` on the Praefect nodes. This was set in the [Praefect](#praefect) section of this guide. This document uses `default` as the Praefect storage name. @@ -1209,6 +1223,110 @@ You can configure: If `default_replication_factor` is unset, the repositories are always replicated on every node defined in `virtual_storages`. If a new node is introduced to the virtual storage, both new and existing repositories are replicated to the node automatically. +## Repository verification + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4080) in GitLab 15.0. + +Praefect stores metadata about the repositories in a database. If the repositories are modified on disk +without going through Praefect, the metadata can become inaccurate. Because the metadata is used for replication +and routing decisions, any inaccuracies may cause problems. Praefect contains a background worker that +periodically verifies the metadata against the actual state on the disks. The worker: + +1. Picks up a batch of replicas to verify on healthy storages. The replicas are either unverified or have exceeded + the configured verification interval. Replicas that have never been verified are prioritized, followed by + the other replicas ordered by longest time since the last successful verification. +1. Checks whether the replicas exist on their respective storages. If the: + - Replica exists, update its last successful verification time. + - Replica doesn't exist, remove its metadata record. + - Check failed, the replica is picked up for verification again when the next worker dequeues more work. + +The worker acquires an exclusive verification lease on each of the replicas it is about to verify. This avoids multiple +workers from verifying the same replica concurrently. The worker releases the leases when it has completed its check. +Praefect contains a background goroutine that releases stale leases every 10 seconds when workers are terminated for +some reason without releasing the lease. + +The worker logs each of the metadata removals prior to executing them. The `perform_deletions` key +indicates whether the invalid metadata records are actually deleted or not. For example: + +```json +{ + "level": "info", + "msg": "removing metadata records of non-existent replicas", + "perform_deletions": false, + "replicas": { + "default": { + "@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git": [ + "praefect-internal-0" + ] + } + } +} +``` + +### Configure the verification worker + +The worker is enabled by default and verifies the metadata records every seven days. The verification +interval is configurable with any valid [Go duration string](https://pkg.go.dev/time#ParseDuration). + +To verify the metadata every three days: + +```ruby +praefect['background_verification_verification_interval'] = '72h' +``` + +Values of 0 and below disable the background verifier. + +```ruby +praefect['background_verification_verification_interval'] = '0' +``` + +#### Enable deletions + +WARNING: +Deletions are disabled by default due to a race condition with repository renames that can cause incorrect +deletions. This is especially prominent in Geo instances as Geo performs more renames than instances without Geo. +See [Handle repository creations, deletions and renames atomically](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4101) +for progress on a fix. We do not recommend enabling the deletions until this is fixed. + +By default, the worker does not delete invalid metadata records but simply logs them and outputs Prometheus +metrics for them. + +You can enable deleting invalid metadata records with: + +```ruby +praefect['background_verification_delete_invalid_records'] = true +``` + +### Prioritize verification manually + +You can prioritize verification of some replicas ahead of their next scheduled verification time. +This might be needed after a disk failure, for example, when the administrator knows that the disk contents may have +changed. Praefect would eventually verify the replicas again, but users may encounter errors in the meantime. + +To manually prioritize reverification of some replicas, use the `praefect verify` subcommand. The subcommand marks +replicas as unverified. Unverified replicas are prioritized by the background verification worker. The verification +worker must be enabled for the replicas to be verified. + +Prioritize verifying the replicas of a specific repository: + +```shell +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml verify -repository-id=<repository-id> +``` + +Prioritize verifying all replicas stored on a virtual storage: + +```shell +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml verify -virtual-storage=<virtual-storage> +``` + +Prioritize verifying all replicas stored on a storage: + +```shell +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml verify -virtual-storage=<virtual-storage> -storage=<storage> +``` + +The output includes the number of replicas that were marked unverified. + ## Automatic failover and primary election strategies Praefect regularly checks the health of each Gitaly node. This is used to automatically fail over diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index a7166f7e62e..6de2acf1792 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Recovery options @@ -348,13 +347,23 @@ If this occurs, run `remove-repository` again. ### Manually list untracked repositories -> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3926) in GitLab 14.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3926) in GitLab 14.4. +> - `older-than` option added in GitLab 15.0. The `list-untracked-repositories` Praefect sub-command lists repositories of the Gitaly Cluster that both: - Exist for at least one Gitaly storage. - Aren't tracked in the Praefect database. +Add the `-older-than` option to avoid showing repositories that are the process of being created and for which a record doesn't yet exist in the +Praefect database. Replace `<duration>` with a time duration (for example, `5s`, `10m`, or `1h`). Defaults to `6h`. + +```shell +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml list-untracked-repositories -older-than <duration> +``` + +Only repositories with a creation time before the specified duration are considered. + The command outputs: - Result to `STDOUT` and the command's logs. diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 9fe09be10a3..d1e802111cd 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Gitaly reference **(FREE SELF)** @@ -71,7 +70,7 @@ Remember to disable `transitioning` when you are done changing your token settings. All authentication attempts are counted in Prometheus under -the [`gitaly_authentications_total` metric](index.md#useful-queries). +the [`gitaly_authentications_total` metric](monitoring.md#useful-queries). ### TLS diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 1be0bf23ed5..c79ed1d1707 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -2,7 +2,6 @@ stage: Create group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Troubleshooting Gitaly and Gitaly Cluster **(FREE SELF)** @@ -356,7 +355,7 @@ The following sections provide possible solutions to Gitaly Cluster errors. ### Check cluster health -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/) in GitLab 14.6. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5688) in GitLab 14.5. The `check` Praefect sub-command runs a series of checks to determine the health of the Gitaly Cluster. @@ -417,6 +416,16 @@ If this check fails: 1. Check if there is a high load on the Praefect database. If the Praefect database is slow to respond, it can lead health checks failing to persist to the database, leading Praefect to think nodes are unhealthy. +#### Check clock synchronization + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4225) in GitLab 14.8. + +Authentication between Praefect and the Gitaly servers requires the server times to be +in sync so the token check succeeds. + +This check helps identify the root cause of `permission denied` +[errors being logged by Praefect](#permission-denied-errors-appearing-in-gitaly-or-praefect-logs-when-accessing-repositories). + ### Praefect errors in logs If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. @@ -513,16 +522,19 @@ Replicas: Generation: 1, fully up to date Healthy: true Valid Primary: true + Verified At: 2021-04-01 10:04:20 +0000 UTC - Storage: "gitaly-2" Assigned: true Generation: 0, behind by 1 changes Healthy: true Valid Primary: false + Verified At: unverified - Storage: "gitaly-3" Assigned: true Generation: replica not yet created Healthy: false Valid Primary: false + Verified At: unverified ``` #### Available metadata @@ -548,6 +560,7 @@ For each replica, the following metadata is available: | `Generation` | Latest confirmed generation of the replica. It indicates:<br><br>- The replica is fully up to date if the generation matches the repository's generation.<br>- The replica is outdated if the replica's generation is less than the repository's generation.<br>- `replica not yet created` if the replica does not yet exist at all on the storage. | | `Healthy` | Indicates whether the Gitaly node that is hosting this replica is considered healthy by the consensus of Praefect nodes. | | `Valid Primary` | Indicates whether the replica is fit to serve as the primary node. If the repository's primary is not a valid primary, a failover occurs on the next write to the repository if there is another replica that is a valid primary. A replica is a valid primary if:<br><br>- It is stored on a healthy Gitaly node.<br>- It is fully up to date.<br>- It is not targeted by a pending deletion job from decreasing replication factor.<br>- It is assigned. | +| `Verified At` | Indicates last successful verification of the replica by the [verification worker](praefect.md#repository-verification). If the replica has not yet been verified, `unverified` is displayed in place of the last successful verification time. Introduced in GitLab 15.0. | ### Check that repositories are in sync diff --git a/doc/administration/inactive_project_deletion.md b/doc/administration/inactive_project_deletion.md new file mode 100644 index 00000000000..a2d2093c57b --- /dev/null +++ b/doc/administration/inactive_project_deletion.md @@ -0,0 +1,60 @@ +--- +stage: Manage +group: Compliance +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Inactive project deletion **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0 [with a flag](../administration/feature_flags.md) named `inactive_projects_deletion`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to +[enable the feature flag](../administration/feature_flags.md) named `inactive_projects_deletion`. +On GitLab.com, this feature is not available. This feature is not ready for production use. + +Administrators of large GitLab instances can find that over time, projects become inactive and are no longer used. +These projects take up unnecessary disk space. With inactive project deletion, you can identify these projects, warn +the maintainers ahead of time, and then delete the projects if they remain inactive. When an inactive project is +deleted, the action generates an audit event that it was performed by the first active administrator. + +## Configure inactive project deletion + +You can configure inactive projects deletion or turn it off using the +[Application settings API](../api/settings.md#change-application-settings). + +The following options are available: + +- `delete_inactive_projects`: Enable or disable inactive project deletion. +- `inactive_projects_min_size_mb`: Minimum size (MB) of inactive projects to be considered for deletion. + Projects smaller in size than this threshold aren't considered inactive. +- `inactive_projects_delete_after_months`: Minimum duration (months) after which a project is scheduled for deletion if + it continues be inactive. +- `inactive_projects_send_warning_email_after_months`: Minimum duration (months) after which a deletion warning email is + sent if a project continues to be inactive. The warning email is sent to users with the Owner and Maintainer roles of + the inactive project. This duration should be less than the `inactive_projects_delete_after_months` duration. + +For example: + +- `delete_inactive_projects` enabled. +- `inactive_projects_min_size_mb` set to `50`. +- `inactive_projects_delete_after_months` set to `12`. +- `inactive_projects_send_warning_email_after_months` set to `6`. + +In this scenario, when a project's size is: + +- Less than 50 MB, the project is not considered inactive. +- Greater than 50 MB and it is inactive for: + - More than 6 months, a deletion warning is email is sent to users with the Owner and Maintainer role on the project + with the scheduled date of deletion. + - More than 12 months, the project is scheduled for deletion. + +## Determine when a project was last active + +You can view a project's activities and determine when the project was last active in the following ways: + +1. Go to the [activity page](../user/project/working_with_projects.md#view-project-activity) for the project and view + the date of the latest event. +1. View the `last_activity_at` attribute for the project using the [Projects API](../api/projects.md). +1. List the visible events for the project using the [Events API](../api/events.md#list-a-projects-visible-events). + View the `created_at` attribute of the latest event. diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 63f8ce3c4cb..2fd57036169 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -302,7 +302,7 @@ gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 -# Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery gitlab_rails['incoming_email_expunge_deleted'] = true ``` @@ -340,7 +340,7 @@ incoming_email: # The IDLE command timeout. idle_timeout: 60 - # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery expunge_deleted: true ``` @@ -384,7 +384,7 @@ gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 -# Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery gitlab_rails['incoming_email_expunge_deleted'] = true ``` @@ -422,7 +422,7 @@ incoming_email: # The IDLE command timeout. idle_timeout: 60 - # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery expunge_deleted: true ``` @@ -463,6 +463,9 @@ gitlab_rails['incoming_email_host'] = "exchange.example.com" gitlab_rails['incoming_email_port'] = 993 # Whether the IMAP server uses SSL gitlab_rails['incoming_email_ssl'] = true + +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` Example for source installs: @@ -491,6 +494,9 @@ incoming_email: port: 993 # Whether the IMAP server uses SSL ssl: true + + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + expunge_deleted: true ``` ##### Dedicated email address @@ -521,6 +527,9 @@ gitlab_rails['incoming_email_host'] = "exchange.example.com" gitlab_rails['incoming_email_port'] = 993 # Whether the IMAP server uses SSL gitlab_rails['incoming_email_ssl'] = true + +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` Example for source installs: @@ -545,6 +554,9 @@ incoming_email: port: 993 # Whether the IMAP server uses SSL ssl: true + + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + expunge_deleted: true ``` #### Microsoft Office 365 @@ -599,6 +611,9 @@ gitlab_rails['incoming_email_host'] = "outlook.office365.com" gitlab_rails['incoming_email_port'] = 993 # Whether the IMAP server uses SSL gitlab_rails['incoming_email_ssl'] = true + +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` This example for source installs assumes the mailbox `incoming@office365.example.com`: @@ -626,6 +641,9 @@ incoming_email: port: 993 # Whether the IMAP server uses SSL ssl: true + + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + expunge_deleted: true ``` ##### Catch-all mailbox @@ -654,6 +672,9 @@ gitlab_rails['incoming_email_host'] = "outlook.office365.com" gitlab_rails['incoming_email_port'] = 993 # Whether the IMAP server uses SSL gitlab_rails['incoming_email_ssl'] = true + +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` This example for source installs assumes the catch-all mailbox `incoming@office365.example.com`: @@ -681,6 +702,9 @@ incoming_email: port: 993 # Whether the IMAP server uses SSL ssl: true + + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + expunge_deleted: true ``` ##### Dedicated email address @@ -708,6 +732,9 @@ gitlab_rails['incoming_email_host'] = "outlook.office365.com" gitlab_rails['incoming_email_port'] = 993 # Whether the IMAP server uses SSL gitlab_rails['incoming_email_ssl'] = true + +# Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +gitlab_rails['incoming_email_expunge_deleted'] = true ``` This example for source installs assumes the dedicated email address `incoming@office365.example.com`: @@ -730,6 +757,9 @@ incoming_email: port: 993 # Whether the IMAP server uses SSL ssl: true + + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + expunge_deleted: true ``` #### Microsoft Graph diff --git a/doc/administration/index.md b/doc/administration/index.md index 73ea4a8e1d0..1d8dcd34d68 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -203,13 +203,8 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Enable Performance Monitoring](monitoring/performance/gitlab_configuration.md): Enable GitLab Performance Monitoring. - [GitLab performance monitoring with Prometheus](monitoring/prometheus/index.md): Configure GitLab and Prometheus for measuring performance metrics. - [GitLab performance monitoring with Grafana](monitoring/performance/grafana_configuration.md): Configure GitLab to visualize time series metrics through graphs and dashboards. - - [Request Profiling](monitoring/performance/request_profiling.md): Get a detailed profile on slow requests. - [Performance Bar](monitoring/performance/performance_bar.md): Get performance information for the current page. -## Analytics - -- [Pseudonymizer](pseudonymizer.md): Export data from a GitLab database to CSV files in a secure way. - ## Troubleshooting - [Debugging tips](troubleshooting/debug.md): Tips to debug problems when things go wrong. diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 0a0c538caa2..8d3fec3b19e 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -636,7 +636,8 @@ If the limit value is set to zero, the limit is disabled. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276192) in GitLab 14.1, disabled by default. > - Enabled by default and [feature flag `ci_jobs_trace_size_limit` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/335259) in GitLab 14.2. -The job log file size limit is 100 megabytes by default. Any job that exceeds this value is dropped. +The job log file size limit in GitLab is 100 megabytes by default. Any job that exceeds the +limit is marked as failed, and dropped by the runner. You can change the limit in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session). Update `ci_jobs_trace_size_limit` with the new value in megabytes: @@ -645,6 +646,11 @@ Update `ci_jobs_trace_size_limit` with the new value in megabytes: Plan.default.actual_limits.update!(ci_jobs_trace_size_limit: 125) ``` +GitLab Runner also has an +[`output_limit` setting](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) +that configures the maximum log size in a runner. Jobs that exceed the runner limit +continue to run, but the log is truncated when it hits the limit. + ### Maximum number of active DAST profile schedules per project > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68551) in GitLab 14.3. @@ -876,6 +882,10 @@ See the limits in the [Add a design to an issue](../user/project/issues/design_m ## Push Event Limits +### Max push size + +The maximum allowed [push size](../user/admin_area/settings/account_and_limit_settings.md#max-push-size) is set to 5 GB. + ### Webhooks and Project Services > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31009) in GitLab 12.4. @@ -886,7 +896,7 @@ than the specified limit, hooks are not executed. More information can be found in these docs: - [Webhooks push events](../user/project/integrations/webhook_events.md#push-events) -- [Project services push hooks limit](../user/project/integrations/overview.md#push-hooks-limit) +- [Project services push hooks limit](../user/project/integrations/index.md#push-hooks-limit) ### Activities diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index 792afc6c3d7..88b320941de 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -6,11 +6,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Web terminals (DEPRECATED) **(FREE)** -> [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + - Read more about the non-deprecated [Web Terminals accessible through the Web IDE](../../user/project/web_ide/index.md). - Read more about the non-deprecated [Web Terminals accessible from a running CI job](../../ci/interactive_web_terminal/index.md). diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 03a0162d526..2582df1b0c4 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -113,8 +113,6 @@ and then `object_store:`. On Omnibus GitLab installs they are prefixed by |---------------------|---------|-------------| | `enabled` | `false` | Enable or disable object storage. | | `remote_directory` | | The bucket name where Artifacts are stored. Use the name only, do not include the path. | -| `direct_upload` | `false` | Set to `true` to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | -| `background_upload` | `true` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3. | | `proxy_download` | `false` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | | `connection` | | Various connection options described below. | @@ -181,67 +179,6 @@ _The artifacts are stored by default in 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. [Migrate any existing local artifacts to the object storage](#migrating-to-object-storage). -### OpenStack example - -See [the available connection settings for OpenStack](object_storage.md#openstack-compatible-connection-settings). - -**In Omnibus installations:** - -_The uploads are stored by default in -`/var/opt/gitlab/gitlab-rails/shared/artifacts`._ - -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting - the values you want: - - ```ruby - gitlab_rails['artifacts_enabled'] = true - gitlab_rails['artifacts_object_store_enabled'] = true - gitlab_rails['artifacts_object_store_remote_directory'] = "artifacts" - gitlab_rails['artifacts_object_store_connection'] = { - 'provider' => 'OpenStack', - 'openstack_username' => 'OS_USERNAME', - 'openstack_api_key' => 'OS_PASSWORD', - 'openstack_temp_url_key' => 'OS_TEMP_URL_KEY', - 'openstack_auth_url' => 'https://auth.cloud.ovh.net', - 'openstack_region' => 'GRA', - 'openstack_tenant_id' => 'OS_TENANT_ID', - } - ``` - -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. [Migrate any existing local artifacts to the object storage](#migrating-to-object-storage). - ---- - -**In installations from source:** - -_The uploads are stored by default in -`/home/git/gitlab/shared/artifacts`._ - -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: - - ```yaml - uploads: - object_store: - enabled: true - direct_upload: false - background_upload: true - proxy_download: false - remote_directory: "artifacts" - connection: - provider: OpenStack - openstack_username: OS_USERNAME - openstack_api_key: OS_PASSWORD - openstack_temp_url_key: OS_TEMP_URL_KEY - openstack_auth_url: 'https://auth.cloud.ovh.net' - openstack_region: GRA - openstack_tenant_id: OS_TENANT_ID - ``` - -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. -1. [Migrate any existing local artifacts to the object storage](#migrating-to-object-storage). - ### Migrating to object storage After [configuring the object storage](#using-object-storage), use the following task to @@ -610,7 +547,7 @@ Bucket names that include folder paths are not supported with [consolidated obje For example, `bucket/path`. If a bucket name has a path in it, you might receive an error similar to: ```plaintext -WARNING: Uploading artifacts as "archive" to coordinator... POST https://gitlab.example.com/api/v4/jobs/job_id/artifacts?artifact_format=zip&artifact_type=archive&expire_in=1+day: 500 Internal Server Error (Missing file) +WARNING: Uploading artifacts as "archive" to coordinator... POST https://gitlab.example.com/api/v4/jobs/job_id/artifacts?artifact_format=zip&artifact_type=archive&expire_in=1+day: 500 Internal Server Error (Missing file) FATAL: invalid argument ``` diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 5c6ea7f52eb..d2837bfa96e 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -177,7 +177,7 @@ Here is the detailed data flow: ### Limitations -- [Redis cluster is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/224171). +- [Redis Cluster is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/224171). - You must configure [object storage for CI/CD artifacts, logs, and builds](job_artifacts.md#object-storage-settings) before you enable the feature flag. After the flag is enabled, files cannot be written to disk, and there is no protection against misconfiguration. diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 3fe6a94ef13..0d75880bdd1 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -89,8 +89,6 @@ The following general settings are supported. |---------------------|-------------|---------| | `enabled` | Enable/disable object storage. | `false` | | `remote_directory` | The bucket name where LFS objects are stored. | | -| `direct_upload` | Set to true to enable direct upload of LFS without the need of local shared storage. Option may be removed after we decide to support only single storage for all files. | `false` | -| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3. | `true` | | `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | `false` | | `connection` | Various connection options described below. | | diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 22e13270179..c89c485e7d3 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -385,7 +385,7 @@ Depending on your installation method, this file is located at: - Omnibus GitLab: `/var/log/gitlab/gitlab-rails/integrations_json.log` - Installations from source: `/home/git/gitlab/log/integrations_json.log` -It contains information about [integration](../user/project/integrations/overview.md) +It contains information about [integration](../user/project/integrations/index.md) activities, such as Jira, Asana, and irker services. It uses JSON format, like this example: @@ -1107,7 +1107,7 @@ in `/var/log/gitlab/gitlab-kas/`. For Omnibus GitLab installations, Praefect logs are in `/var/log/gitlab/praefect/`. -GitLab also tracks [Prometheus metrics for Praefect](gitaly/#monitor-gitaly-cluster). +GitLab also tracks [Prometheus metrics for Praefect](gitaly/monitoring.md#monitor-gitaly-cluster). ## Backup log diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index 50c0f0ecc63..41eed500594 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -116,11 +116,11 @@ For most JSON requests, POST, PUT, PATCH, and DELETE are blocked, and the API re | POST | `/users/sign_in` | To allow users to log in. | | POST | `/users/sign_out`| To allow users to log out. | | POST | `/oauth/token` | To allow users to log in to a Geo secondary for the first time. | -| POST | `/admin/session`, `/admin/session/destroy` | To allow [Administrator mode for GitLab administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158) | +| POST | `/admin/session`, `/admin/session/destroy` | To allow [Admin Mode for GitLab administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158) | | POST | Paths ending with `/compare`| Git revision routes. | | POST | `.git/git-upload-pack` | To allow Git pull/clone. | | POST | `/api/v4/internal` | [internal API routes](../../development/internal_api/index.md) | -| POST | `/admin/sidekiq` | To allow management of background jobs in the Admin UI | +| POST | `/admin/sidekiq` | To allow management of background jobs in the Admin Area | | POST | `/admin/geo` | To allow updating Geo Nodes in the administrator UI | | POST | `/api/v4/geo_replication`| To allow certain Geo-specific administrator UI actions on secondary sites | @@ -166,7 +166,7 @@ Package Registry allows you to install but not publish packages. Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled. -[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node), +[During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site), it is recommended that you disable all cron jobs except for those related to Geo. To monitor queues and disable jobs: @@ -210,4 +210,4 @@ For the same reason we don't automatically block background jobs when Maintenanc The resulting database writes are acceptable. Here, the trade-off is between more service degradation and the completion of replication. -However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-node). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal. +However, during a planned failover, we [ask users to turn off cron jobs that are not related to Geo, manually](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site). In the absence of new database writes and non-Geo cron jobs, new background jobs would either not be created at all or be minimal. diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 01576eb4abf..fe1c74b0e24 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -117,8 +117,6 @@ then `object_store:`. On Omnibus installations, they are prefixed by |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where external diffs are stored| | -| `direct_upload` | Set to `true` to enable direct upload of external diffs without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` | -| `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 | `true` | | `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | diff --git a/doc/administration/monitoring/performance/img/request_profile_result.png b/doc/administration/monitoring/performance/img/request_profile_result.png Binary files differdeleted file mode 100644 index 9176a0b49fd..00000000000 --- a/doc/administration/monitoring/performance/img/request_profile_result.png +++ /dev/null diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 20fad8baf91..b063a20dc7f 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -17,7 +17,6 @@ documents to understand and properly configure GitLab Performance Monitoring: - [Prometheus documentation](../prometheus/index.md) - [Grafana Install/Configuration](grafana_configuration.md) - [Performance bar](performance_bar.md) -- [Request profiling](request_profiling.md) ## Introduction to GitLab Performance Monitoring diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index 6cd20132092..d0d05c16ea0 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -2,43 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-07-26' +redirect_to: 'index.md' --- -# Request profiling (DEPRECATED) **(FREE SELF)** +# Request profiling (removed) **(FREE SELF)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352488) in GitLab 14.8, and planned for removal in GitLab 15.0. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352488) -in GitLab 14.8, and is planned for removal in GitLab 15.0. - -To profile a request: - -1. Sign in to GitLab as an Administrator or a user with the [Maintainer role](../../../user/permissions.md). -1. In the navigation bar, click **Admin area**. -1. Go to **Monitoring > Requests Profiles**. -1. In the **Requests Profiles** section, copy the token. -1. Pass the headers `X-Profile-Token: <token>` and `X-Profile-Mode: <mode>`(where - `<mode>` can be `execution` or `memory`) to the request you want to profile. When - passing headers, you can use: - - - Browser extensions such as the - [ModHeader](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj) - Chrome extension. - - `curl`. For example: - - ```shell - curl --header 'X-Profile-Token: <token>' --header 'X-Profile-Mode: <mode>' "https://gitlab.example.com/group/project" - ``` - - Profiled requests can take longer than usual. - -After the request completes, you can view the profiling output from the -**Monitoring > Requests Profiles** administration page: - - - -## Cleaning up profiled requests - -The output from profiled requests is cleared out once each day through a -Sidekiq worker. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352488) in GitLab 14.8 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350152) in 15.0. diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 08f7d5095da..4f8fbd0c07e 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -156,7 +156,7 @@ The following metrics can be controlled by feature flags: ## Praefect metrics You can [configure Praefect](../../gitaly/praefect.md#praefect) to report metrics. For information -on available metrics, see the [relevant documentation](../../gitaly/index.md#monitor-gitaly-cluster). +on available metrics, see the [relevant documentation](../../gitaly/monitoring.md#monitor-gitaly-cluster). ## Sidekiq metrics diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index ef5d26ac845..b0c50d63e78 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -24,9 +24,9 @@ file system performance, see Starting with GitLab version 14.0, support for NFS to store Git repository data is deprecated. Technical customer support and engineering support is available for the 14.x releases. Engineering is fixing bugs and security vulnerabilities consistent with our [release and maintenance policy](../policy/maintenance.md#security-releases). -Upon the release of GitLab 15.0 (tentatively May 22nd, 2022) technical and engineering support for using NFS to store Git repository data will be officially at end-of-life. There will be no product changes or troubleshooting provided via Engineering, Security or Paid Support channels after the release date of 15.0, regardless of your GitLab version. +Upon the release of GitLab 15.6 technical and engineering support for using NFS to store Git repository data will be officially at end-of-life. There will be no product changes or troubleshooting provided via Engineering, Security or Paid Support channels after the release date of 15.6, regardless of your GitLab version. -Until the release of 15.0, for customers running 14.x releases, we continue to help with Git related tickets from customers running one or more Gitaly servers with its data stored on NFS. Examples may include: +Until the release of 15.6, for customers running 14.x releases, we continue to help with Git related tickets from customers running one or more Gitaly servers with its data stored on NFS. Examples may include: - Performance issues or timeouts accessing Git data - Commits or branches vanish @@ -39,10 +39,10 @@ Assistance is limited to activities like: - Verifying that NFS client mount options match our [documented recommendations](#mount-options) - Analyzing the GitLab Workhorse and Rails logs, and determining that `500` errors being seen in the environment are caused by slow responses from Gitaly -GitLab support is unable to continue with the investigation if: +GitLab support is unable to continue with the investigation if both: -- The date of the request is on or after the release of GitLab version 15.0, and -- Support Engineers and Management determine that all reasonable non-NFS root causes have been exhausted +- The date of the request is on or after the release of GitLab version 15.6. +- Support Engineers and Management determine that all reasonable non-NFS root causes have been exhausted. If the issue is reproducible, or if it happens intermittently but regularly, GitLab Support can investigate providing the issue reproduces without the use of NFS. In order to reproduce without NFS, the affected repositories should be migrated to a different Gitaly shard, such as Gitaly cluster or a standalone Gitaly VM, backed with block storage. @@ -377,7 +377,7 @@ Any `Operation not permitted` errors means you should investigate your NFS serve ## NFS in a Firewalled Environment -If the traffic between your NFS server and NFS client(s) is subject to port filtering +If the traffic between your NFS server and NFS clients is subject to port filtering by a firewall, then you need to reconfigure that firewall to allow NFS communication. [This guide from The Linux Documentation Project (TDLP)](https://tldp.org/HOWTO/NFS-HOWTO/security.html#FIREWALLS) @@ -482,3 +482,10 @@ On Ubuntu 16.04, use: ```shell sudo perf trace --no-syscalls --event 'nfs4:*' -p $(pgrep -fd ',' puma) ``` + +### Warnings `garbage found: .../repositories/@hashed/...git/objects/pack/.nfs...` in Gitaly logs + +If you find any warnings like `garbage found: .../repositories/@hashed/...git/objects/pack/.nfs...` in [Gitaly logs](logs.md#gitaly-logs), +this problem occurs if `lookupcache=positive` is not set, which we recommend as an +[NFS mount option](#mount-options). +See [Gitaly issue #3175](https://gitlab.com/gitlab-org/gitaly/-/issues/3175) for more details. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 6ea433f95e9..0560a8813df 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -19,7 +19,7 @@ GitLab has been tested by vendors and customers on a number of object storage pr - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatible mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors, whose list is not officially established. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -62,7 +62,7 @@ Using the consolidated object storage configuration has a number of advantages: - It enables the use of [encrypted S3 buckets](#encrypted-s3-buckets). - It [uploads files to S3 with proper `Content-MD5` headers](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). -Because [direct upload mode](../development/uploads/implementation.md#direct-upload) +Because [direct upload mode](../development/uploads/index.md#direct-upload) must be enabled, only the following providers can be used: - [Amazon S3-compatible providers](#s3-compatible-connection-settings) @@ -386,52 +386,6 @@ If you are using a custom Azure storage domain, configuration. This information is exchanged in an API call between GitLab Rails and Workhorse. -#### OpenStack-compatible connection settings - -Although OpenStack Swift provides S3 compatibility, some users may want to use -the [Swift API](https://docs.openstack.org/swift/latest/api/object_api_v1_overview.html). - -This isn't compatible with the consolidated object storage form. OpenStack Swift -is supported only with the storage-specific form. If you want to use the -consolidated form, see the [S3 settings](#s3-compatible-connection-settings). - -Here are the valid connection settings for the Swift API, provided by -[fog-openstack](https://github.com/fog/fog-openstack): - -| Setting | Description | Default | -|--------------------------|----------------------|---------| -| `provider` | Always `OpenStack` for compatible hosts. | `OpenStack` | -| `openstack_username` | OpenStack username. | | -| `openstack_api_key` | OpenStack API key. | | -| `openstack_temp_url_key` | OpenStack key for generating temporary URLs | | -| `openstack_auth_url` | OpenStack authentication endpoint | | -| `openstack_region` | OpenStack region. | | -| `openstack_tenant` | OpenStack tenant ID. | | - -#### Rackspace Cloud Files - -The following table describes the valid connection parameters for -Rackspace Cloud, provided by [fog-rackspace](https://github.com/fog/fog-rackspace/). - -This isn't compatible with the consolidated object storage form. -Rackspace Cloud is supported only with the storage-specific form. - -| Setting | Description | Example | -|--------------------------|----------------|-------------| -| `provider` | Provider name. | `Rackspace` | -| `rackspace_username` | Username of the Rackspace account with access to the container. | `joe.smith` | -| `rackspace_api_key` | API key of the Rackspace account with access to the container. | `ABC123DEF456ABC123DEF456ABC123DE` | -| `rackspace_region` | Rackspace storage region to use, a three letter code from the [list of service access endpoints](https://docs.rackspace.com/docs/cloud-files/v1/general-api-info/service-access/). | `iad` | -| `rackspace_temp_url_key` | Private key you set in the Rackspace API for [temporary URLs](https://docs.rackspace.com/docs/cloud-files/v1/use-cases/public-access-to-your-cloud-files-account/#tempurl). | `ABC123DEF456ABC123DEF456ABC123DE` | - -Regardless of whether the container has public access enabled or disabled, Fog -uses the TempURL method to grant access to LFS objects. If you see error -messages in logs that refer to instantiating storage with a `temp-url-key`, -be sure you have set the key properly both in the Rackspace API and in -`gitlab.rb`. You can verify the value of the key Rackspace has set by sending a -GET request with token header to the service access endpoint URL and comparing -the output of the returned headers. - ### Object-specific configuration The following YAML shows how the `object_store` section defines @@ -582,7 +536,6 @@ supported by consolidated configuration form, refer to the following guides: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| **{dotted-circle}** No | | [Packages](packages/index.md#using-object-storage) (optional feature) | **{check-circle}** Yes | | [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) | **{check-circle}** Yes | -| [Pseudonymizer](pseudonymizer.md) (optional feature) | **{dotted-circle}** No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | | [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | | [Pages content](pages/index.md#using-object-storage) | **{check-circle}** Yes | @@ -616,7 +569,7 @@ There are plans to [enable the use of a single bucket](https://gitlab.com/gitlab in the future. Helm-based installs require separate buckets to -[handle backup restorations](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer). +[handle backup restorations](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-terraform-state-dependency-proxy). ### S3 API compatibility issues @@ -799,3 +752,46 @@ to run the following command: ```ruby Feature.disable(:s3_multithreaded_uploads) ``` + +## Migrate objects to a different object storage provider + +You may need to migrate GitLab data in object storage to a different object storage provider. The following steps show you how do this using [Rclone](https://rclone.org/). + +The steps assume you are moving the `uploads` bucket, but the same process works for other buckets. + +Prerequisites: + +- Choose the computer to run Rclone on. Depending on how much data you are migrating, Rclone may have to run for a long time so you should avoid using a laptop or desktop computer that can go into power saving. You can use your GitLab server to run Rclone. + +1. [Install](https://rclone.org/downloads/) Rclone. +1. Configure Rclone by running the following: + + ```shell + rclone config + ``` + + The configuration process is interactive. Add at least two "remotes": one for the object storage provider your data is currently on (`old`), and one for the provider you are moving to (`new`). + +1. Verify that you can read the old data. The following example refers to the `uploads` bucket , but your bucket may have a different name: + + ```shell + rclone ls old:uploads | head + ``` + + This should print a partial list of the objects currently stored in your `uploads` bucket. If you get an error, or if + the list is empty, go back and update your Rclone configuration using `rclone config`. + +1. Perform an initial copy. You do not need to take your GitLab server offline for this step. + + ```shell + rclone sync -P old:uploads new:uploads + ``` + +1. After the first sync completes, use the web UI or command-line interface of your new object storage provider to + verify that there are objects in the new bucket. If there are none, or if you encounter an error while running `rclone + sync`, check your Rclone configuration and try again. + +After you have done at least one successful Rclone copy from the old location to the new location, schedule maintenance and take your GitLab server offline. During your maintenance window you must do two things: + +1. Perform a final `rclone sync` run, knowing that your users cannot add new objects so you will not leave any behind in the old bucket. +1. Update the object storage configuration of your GitLab server to use the new provider for `uploads`. diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index fd04efe8473..75f2ad5ed26 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -354,7 +354,7 @@ file is written, but this can be changed by passing the `--pidfile` option to ``` Keep in mind that the PID file contains the PID of the `sidekiq-cluster` -command and not the PID(s) of the started Sidekiq processes. +command and not the PIDs of the started Sidekiq processes. ### Environment diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md index bb8eb184302..cd3a53b7c63 100644 --- a/doc/administration/operations/extra_sidekiq_routing.md +++ b/doc/administration/operations/extra_sidekiq_routing.md @@ -88,8 +88,8 @@ components: > [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). -Queue matching query works upon the worker attributes, described in [Sidekiq -style guide](../../development/sidekiq_style_guide.md). We support querying +Queue matching query works upon the worker attributes, described in +[Sidekiq style guide](../../development/sidekiq/index.md). We support querying based on a subset of worker attributes: - `feature_category` - the [GitLab feature diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 477a11dd899..a3240a6041b 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -1,6 +1,6 @@ --- -stage: Enablement -group: Distribution +stage: Create +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -14,7 +14,7 @@ drop-in replacement. Regular SSH operations become slow as the number of users grows because OpenSSH searches for a key to authorize a user via a linear search. In the worst case, -such as when the user is not authorized to access GitLab, OpenSSH will scan the +such as when the user is not authorized to access GitLab, OpenSSH scans the entire file to search for a key. This can take significant time and disk I/O, which delays users attempting to push or pull to a repository. Making matters worse, if users add or remove keys frequently, the operating system may @@ -33,21 +33,22 @@ able to accept a fingerprint. Check the version of OpenSSH on your server with ` Unlike [Cloud Native GitLab](https://docs.gitlab.com/charts/), Omnibus GitLab by default manages an `authorized_keys` file that is located in the -`git` user's home directory. For most installations, this will be located under -`/var/opt/gitlab/.ssh/authorized_keys`, but you can use the following command to locate the `authorized_keys` on your system: +`git` user's home directory. For most installations, this file is located under +`/var/opt/gitlab/.ssh/authorized_keys`, but you can use the following command to +locate the `authorized_keys` on your system: ```shell getent passwd git | cut -d: -f6 | awk '{print $1"/.ssh/authorized_keys"}' ``` The `authorized_keys` file contains all the public SSH keys for users allowed to access GitLab. However, to maintain a -single source of truth, [Geo](../geo/index.md) needs to be configured to perform SSH fingerprint +single source of truth, [Geo](../geo/index.md) must be configured to perform SSH fingerprint lookups via database lookup. As part of [setting up Geo](../geo/index.md#setup-instructions), you are required to follow the steps outlined below for both the primary and -secondary nodes, but the `Write to "authorized keys" file` checkbox -only needs to be unchecked on the primary node since it is reflected +secondary nodes, but **Write to "authorized keys" file** +must be unchecked only on the primary node, because it is reflected automatically on the secondary if database replication is working. ## Setting up fast lookup via GitLab Shell @@ -56,8 +57,8 @@ GitLab Shell provides a way to authorize SSH users via a fast, indexed lookup to the GitLab database. GitLab Shell uses the fingerprint of the SSH key to check whether the user is authorized to access GitLab. -Add the following to your `sshd_config` file. This is usually located at -`/etc/ssh/sshd_config`, but it will be `/assets/sshd_config` if you're using +Add the following to your `sshd_config` file. This file is usually located at +`/etc/ssh/sshd_config`, but it is at `/assets/sshd_config` if you're using Omnibus Docker: ```plaintext @@ -84,17 +85,14 @@ file (start the line with a `#` to comment it), and from your local machine, att ssh -T git@gitlab.example.com ``` -A successful pull or [welcome message](../../user/ssh.md#verify-that-you-can-connect) would mean that GitLab was able to find the key in the database, -since it is not present in the file anymore. - -NOTE: -For Omnibus Docker, `AuthorizedKeysCommand` is setup by default in -GitLab 11.11 and later. +A successful pull or [welcome message](../../user/ssh.md#verify-that-you-can-connect) +means that GitLab was able to find the key in the database, +as it is not present in the file. NOTE: For Installations from source, the command would be located at `/home/git/gitlab-shell/bin/gitlab-shell-authorized-keys-check` if [the install from source](../../install/installation.md#install-gitlab-shell) instructions were followed. -You might want to consider creating a wrapper script somewhere else since this command needs to be +You might want to consider creating a wrapper script somewhere else, as this command must be owned by `root` and not be writable by group or others. You could also consider changing the ownership of this command as required, but that might require temporary ownership changes during `gitlab-shell` upgrades. @@ -123,7 +121,7 @@ or for users to re-add their keys. ## How to go back to using the `authorized_keys` file -This is a brief overview. Please refer to the above instructions for more context. +This overview is brief. Refer to the above instructions for more context. 1. [Rebuild the `authorized_keys` file](../raketasks/maintenance.md#rebuild-authorized_keys-file). 1. Enable writes to the `authorized_keys` file. @@ -136,27 +134,35 @@ This is a brief overview. Please refer to the above instructions for more contex ## Use `gitlab-sshd` instead of OpenSSH -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299109) in GitLab 14.5. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299109) in GitLab 14.5 as an **Alpha** release for self-managed customers. WARNING: `gitlab-sshd` is in [**Alpha**](../../policy/alpha-beta-support.md#alpha-features). It is not ready for production use. `gitlab-sshd` is [a standalone SSH server](https://gitlab.com/gitlab-org/gitlab-shell/-/tree/main/internal/sshd) - written in Go. It is provided as a part of `gitlab-shell` package. It has a lower memory - use as a OpenSSH alternative and supports - [group access restriction by IP address](../../user/group/index.md) for applications - running behind the proxy. +written in Go. It is provided as a part of the `gitlab-shell` package. It has a lower memory +use as a OpenSSH alternative, and supports +[group access restriction by IP address](../../user/group/index.md) for applications +running behind the proxy. + +`gitlab-sshd` is a lightweight alternative to OpenSSH for providing +[SSH operations](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/71a7f34a476f778e62f8fe7a453d632d395eaf8f/doc/features.md). +While OpenSSH uses a restricted shell approach, `gitlab-sshd` behaves more like a +modern multi-threaded server application, responding to incoming requests. The major +difference is that OpenSSH uses SSH as a transport protocol while `gitlab-sshd` uses Remote Procedure Calls (RPCs). + +The capabilities of GitLab Shell are not limited to Git operations. If you are considering switching from OpenSSH to `gitlab-sshd`, consider these concerns: - The `gitlab-sshd` component is only available for [Cloud Native Helm Charts](https://docs.gitlab.com/charts/) deployments. - `gitlab-sshd` supports the PROXY protocol. It can run behind proxy servers that rely - on it, such as HAProxy. -- `gitlab-sshd` does not share a SSH port with the system administrator's OpenSSH, - and requires a bind to port 22. -- `gitlab-sshd` **does not** support SSH certificates. + on it, such as HAProxy. The PROXY protocol not enabled by default, but can be enabled with a Helm chart setting. +- By default, `gitlab-sshd` binds to port 22, but you can configure a different port in the Helm chart. +- `gitlab-sshd` **does not** support SSH certificates. For more details, read + [issue #495](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/495). To switch from OpenSSH to `gitlab-sshd`: @@ -178,7 +184,11 @@ GitLab supports `authorized_keys` database lookups with [SELinux](https://en.wik Because the SELinux policy is static, GitLab doesn't support the ability to change internal webserver ports at the moment. Administrators would have to create a special `.te` -file for the environment, since it isn't generated dynamically. +file for the environment, as it isn't generated dynamically. + +### Additional documentation + +Additional technical documentation for `gitlab-sshd` may be found on the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/README.md) documentation page. ## Troubleshooting @@ -187,7 +197,8 @@ or causing high CPU load, be sure to check the size of `/var/log/btmp`, and ensu If this file is very large, GitLab SSH fast lookup can cause the bottleneck to be hit more frequently, thus decreasing performance even further. If you are able to, you may consider disabling [`UsePAM` in your `sshd_config`](https://linux.die.net/man/5/sshd_config) to avoid reading `/var/log/btmp` altogether. -Running `strace` and `lsof` on a running `sshd: git` process can return useful debugging information. To get an `strace` on an in-progress Git over SSH connection for IP `x.x.x.x`, run: +Running `strace` and `lsof` on a running `sshd: git` process returns debugging information. +To get an `strace` on an in-progress Git over SSH connection for IP `x.x.x.x`, run: ```plaintext sudo strace -s 10000 -p $(sudo netstat -tp | grep x.x.x.x | egrep 'ssh.*: git' | sed -e 's/.*ESTABLISHED *//' -e 's#/.*##') diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index c12f75989c3..12a8b2faadc 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -179,6 +179,46 @@ optimal configuration: - To force Rugged to be used with multi-threaded Puma, you can use a [feature flag](../../development/gitaly.md#legacy-rugged-code). +## Configuring Puma to listen over SSL + +Puma, when deployed with Omnibus GitLab, listens over a Unix socket by +default. To configure Puma to listen over an HTTPS port instead, follow the +steps below: + +1. Generate an SSL certificate key-pair for the address where Puma will + listen. For the example below, this is `127.0.0.1`. + + NOTE: + If using a self-signed certificate from a custom Certificate Authority (CA), + follow [the documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) + to make them trusted by other GitLab components. + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + puma['ssl_listen'] = '127.0.0.1' + puma['ssl_port'] = 9111 + puma['ssl_certificate'] = '<path_to_certificate>' + puma['ssl_certificate_key'] = '<path_to_key>' + + # Disable UNIX socket + puma['socket'] = "" + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +NOTE: +In addition to the Unix socket, Puma also listens over HTTP on port 8080 for +providing metrics to be scraped by Prometheus. It is not currently possible to +make Prometheus scrape them over HTTPS, and support for it is being discussed +[in this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6811). +Hence, it is not technically possible to turn off this HTTP listener without +losing Prometheus metrics. + ## Switch from Unicorn to Puma NOTE: diff --git a/doc/administration/package_information/licensing.md b/doc/administration/package_information/licensing.md index a7bf5c52d7b..d27c1df0ccf 100644 --- a/doc/administration/package_information/licensing.md +++ b/doc/administration/package_information/licensing.md @@ -66,7 +66,7 @@ This software is based in part on the work of the Independent JPEG Group. ## Trademark Usage -Within the GitLab documentation, reference to third party technology(ies) and/or trademarks of third party entities, may be made. The inclusion of reference to third party technology and/or entities is solely for the purposes of example(s) of how GitLab software may interact with, or be used in conjunction with, such third party technology. +Within the GitLab documentation, reference to third-party technologies and/or trademarks of third-party entities may be made. The inclusion of reference to third-party technology and/or entities is solely for the purposes of examples of how GitLab software may interact with, or be used in conjunction with, such third-party technology. All trademarks, materials, documentation, and other intellectual property remain the property of any/all such third party. ### Trademark Requirements diff --git a/doc/administration/package_information/postgresql_versions.md b/doc/administration/package_information/postgresql_versions.md index c80437221c4..97f93663003 100644 --- a/doc/administration/package_information/postgresql_versions.md +++ b/doc/administration/package_information/postgresql_versions.md @@ -26,6 +26,7 @@ Read more about update policies and warnings in the PostgreSQL | GitLab version | PostgreSQL versions | Default version for fresh installs | Default version for upgrades | Notes | | -------------- | --------------------- | ---------------------------------- | ---------------------------- | ----- | +| 15.0 | 12.10, 13.6 | 13.6 | 13.6 | Users can manually upgrade to 13.6 following the [upgrade docs](https://docs.gitlab.com/omnibus/settings/database.html#gitlab-150-and-later). | | 14.1 | 12.7, 13.3 | 12.7 | 12.7 | PostgreSQL 13 available for fresh installations if not using [Geo](../geo/index.md#requirements-for-running-geo) or [Patroni](../postgresql/index.md#postgresql-replication-and-failover-with-omnibus-gitlab). | 14.0 | 12.7 | 12.7 | 12.7 | HA installations with repmgr are no longer supported and will be prevented from upgrading to Omnibus GitLab 14.0 | | 13.8 | 11.9, 12.4 | 12.4 | 12.4 | Package upgrades automatically performed PostgreSQL upgrade for nodes that are not part of a Geo or HA cluster.). | diff --git a/doc/administration/package_information/signed_packages.md b/doc/administration/package_information/signed_packages.md index d7bcfa113ff..24857dcfc27 100644 --- a/doc/administration/package_information/signed_packages.md +++ b/doc/administration/package_information/signed_packages.md @@ -14,7 +14,7 @@ These packages are produced by the GitLab CI process, as found in the [Omnibus ## GnuPG Public Keys -All packages are signed with [GnuPG](https://www.gnupg.org/), in a method appropriate for their format. The key used to sign these packages can be found on [pgp.mit.edu](https://pgp.mit.edu) at [0x3cfcf9baf27eab47](https://pgp.mit.edu/pks/lookup?op=vindex&search=0x3CFCF9BAF27EAB47) +All packages are signed with [GnuPG](https://www.gnupg.org/), in a method appropriate for their format. The key used to sign these packages can be found on [MIT PGP Public Key Server](https://pgp.mit.edu) at [0x3cfcf9baf27eab47](https://pgp.mit.edu/pks/lookup?op=vindex&search=0x3CFCF9BAF27EAB47) ## Verifying Signatures diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index d6f5588ebc9..0edad83cc18 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -158,7 +158,7 @@ If your certificate provider provides the CA Bundle certificates, append them to An administrator may want the container registry listening on an arbitrary port such as `5678`. However, the registry and application server are behind an AWS application load balancer that only -listens on ports `80` and `443`. The admin may simply remove the port number for +listens on ports `80` and `443`. The administrator may simply remove the port number for `registry_external_url`, so HTTP or HTTPS is assumed. Then, the rules apply that map the load balancer to the registry from ports `80` or `443` to the arbitrary port. This is important if users rely on the `docker login` example in the container registry. Here's an example: @@ -1244,9 +1244,9 @@ unauthorized: authentication required GitLab has a default token expiration of 5 minutes for the registry. When pushing larger images, or images that take longer than 5 minutes to push, users may -encounter this error. +encounter this error. On GitLab.com, the expiration time is 15 minutes. -Administrators can increase the token duration in **Admin area > Settings > +Administrators can increase the token duration in **Admin Area > Settings > CI/CD > Container Registry > Authorization token duration (minutes)**. ### Docker login attempt fails with: 'token signed by untrusted key' diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index 4c50cfeeb0f..40c1b9d795a 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -140,8 +140,6 @@ This section describes the earlier configuration format. gitlab_rails['dependency_proxy_storage_path'] = "/var/opt/gitlab/gitlab-rails/shared/dependency_proxy" gitlab_rails['dependency_proxy_object_store_enabled'] = true gitlab_rails['dependency_proxy_object_store_remote_directory'] = "dependency_proxy" # The bucket name. - gitlab_rails['dependency_proxy_object_store_direct_upload'] = false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - gitlab_rails['dependency_proxy_object_store_background_upload'] = true # Temporary option to limit automatic upload (Default: true). gitlab_rails['dependency_proxy_object_store_proxy_download'] = false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. gitlab_rails['dependency_proxy_object_store_connection'] = { ## @@ -177,8 +175,6 @@ This section describes the earlier configuration format. object_store: enabled: false remote_directory: dependency_proxy # The bucket name. - # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - # background_upload: true # Temporary option to limit automatic upload (Default: true). # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. connection: ## @@ -248,23 +244,6 @@ Verify that there are no files on disk in the `dependency_proxy` folder: sudo find /var/opt/gitlab/gitlab-rails/shared/dependency_proxy -type f | grep -v tmp | wc -l ``` -## Disabling Authentication - -Authentication was introduced in 13.7 as part of [enabling private groups to use the -Dependency Proxy](https://gitlab.com/gitlab-org/gitlab/-/issues/11582). If you -previously used the Dependency Proxy without authentication and need to disable -this feature while you update your workflow to [authenticate with the Dependency -Proxy](../../user/packages/dependency_proxy/index.md#authenticate-with-the-dependency-proxy), -the following commands can be issued in a Rails console: - -```ruby -# Disable the authentication -Feature.disable(:dependency_proxy_for_private_groups) - -# Re-enable the authentication -Feature.enable(:dependency_proxy_for_private_groups) -``` - ## Changing the JWT expiration The Dependency Proxy follows the [Docker v2 token authentication flow](https://docs.docker.com/registry/spec/auth/token/), @@ -280,7 +259,7 @@ ApplicationSetting.update(container_registry_token_expire_delay: 30) The default expiration and the expiration on GitLab.com is 15 minutes. ## Using the dependency proxy behind a proxy - + 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines: ```ruby diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index b122cb9db90..ef00127a70e 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -152,8 +152,6 @@ We recommend using the [consolidated object storage settings](../object_storage. gitlab_rails['packages_enabled'] = true gitlab_rails['packages_object_store_enabled'] = true gitlab_rails['packages_object_store_remote_directory'] = "packages" # The bucket name. - gitlab_rails['packages_object_store_direct_upload'] = false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - gitlab_rails['packages_object_store_background_upload'] = true # Temporary option to limit automatic upload (Default: true). gitlab_rails['packages_object_store_proxy_download'] = false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. gitlab_rails['packages_object_store_connection'] = { ## @@ -192,8 +190,6 @@ We recommend using the [consolidated object storage settings](../object_storage. object_store: enabled: false remote_directory: packages # The bucket name. - # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false). - # background_upload: true # Temporary option to limit automatic upload (Default: true). # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage. connection: ## diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 114894c01f0..f144c60fcfe 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -97,10 +97,9 @@ IPv6 address. If you don't have IPv6, you can omit the `AAAA` record. #### DNS configuration for custom domains -If support for custom domains is needed, the Pages root domain and its subdomains should point to -the secondary IP (which is dedicated for the Pages daemon). `<namespace>.<pages root domain>` should -point at Pages directly. Without this, users aren't able to use `CNAME` records to point their -custom domains to their GitLab Pages. +If support for custom domains is needed, all subdomains of the Pages root domain should point to the +secondary IP (which is dedicated for the Pages daemon). Without this configuration, users can't use +`CNAME` records to point their custom domains to their GitLab Pages. For example, an entry could look like this: @@ -295,7 +294,7 @@ control over how the Pages daemon runs and serves content in your environment. | `rate_limit_domain_burst` | Rate limit per domain maximum burst allowed per second. | | `server_read_timeout` | Maximum duration to read the request headers and body. For no timeout, set to `0` or a negative value. Default: `5s` | | `server_read_header_timeout` | Maximum duration to read the request headers. For no timeout, set to `0` or a negative value. Default: `1s` | -| `server_write_timeout` | Maximum duration to write all files in the response. Larger files require more time. For no timeout, set to `0` or a negative value. Default: `5m` | +| `server_write_timeout` | Maximum duration to write all files in the response. Larger files require more time. For no timeout, set to `0` or a negative value. Default: `0` | | `server_keep_alive` | The `Keep-Alive` period for network connections accepted by this listener. If `0`, `Keep-Alive` is enabled if supported by the protocol and operating system. If negative, `Keep-Alive` is disabled. Default: `15s` | ## Advanced configuration @@ -387,6 +386,12 @@ GitLab supports [custom domain verification](../../user/project/pages/custom_dom When adding a custom domain, users are required to prove they own it by adding a GitLab-controlled verification code to the DNS records for that domain. +WARNING: +Disabling domain verification is unsafe and can lead to various vulnerabilities. +If you *do* disable it, either ensure that the Pages root domain itself does not point to the +secondary IP or add the root domain as custom domain to a project; otherwise, any user can add this +domain as a custom domain to their project. + If your user base is private or otherwise trusted, you can disable the verification requirement: @@ -1135,7 +1140,7 @@ Rate limits are enforced using the following: - `rate_limit_source_ip`: Set the maximum threshold in number of requests per client IP per second. Set to 0 to disable this feature. - `rate_limit_source_ip_burst`: Sets the maximum threshold of number of requests allowed in an initial outburst of requests per client IP. For example, when you load a web page that loads a number of resources at the same time. -- `rate_limit_domain_ip`: Set the maximum threshold in number of requests per hosted pages domain per second. Set to 0 to disable this feature. +- `rate_limit_domain`: Set the maximum threshold in number of requests per hosted pages domain per second. Set to 0 to disable this feature. - `rate_limit_domain_burst`: Sets the maximum threshold of number of requests allowed in an initial outburst of requests per hosted pages domain. #### Enable source-IP rate limits @@ -1339,6 +1344,27 @@ To stop `systemd` from cleaning the Pages related content: sudo gitlab-ctl restart gitlab-pages ``` +### Unable to access GitLab Pages + +If you can't access your GitLab Pages (such as receiving `502 Bad Gateway` errors, or a login loop) +and in your Pages log shows this error: + +```plaintext +"error":"retrieval context done: context deadline exceeded","host":"root.docs-cit.otenet.gr","level":"error","msg":"could not fetch domain information from a source" +``` + +1. Add the following to `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['internal_gitlab_server'] = 'http://localhost:8080' + ``` + +1. Restart GitLab Pages: + + ```shell + sudo gitlab-ctl restart gitlab-pages + ``` + ### 404 error after promoting a Geo secondary to a primary node Pages files are not among the diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index a666c1fab95..ed3c662eba3 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -219,12 +219,12 @@ the database. Each of the listed services below use the following formula to def - `puma` : `max_threads + headroom` (default `14`) - `max_threads` is configured via: `gitlab['puma']['max_threads']` (default: `4`) - - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) + - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) - `sidekiq` : `max_concurrency + 1 + headroom` (default: `61`) - `max_concurrency` is configured via: `sidekiq['max_concurrency']` (default: `50`) - - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) + - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) - `geo-logcursor`: `1+headroom` (default: `11`) - - `headroom` can be configured via `DB_POOL_HEADROOM` env variable (default to `10`) + - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) To calculate the `default_pool_size`, multiply the number of instances of `puma`, `sidekiq` and `geo-logcursor` by the number of connections each can consume as per listed above. The total will be the suggested `default_pool_size`. diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index 8c7151606a5..84122149cb8 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1123,25 +1123,36 @@ postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) ### Reinitialize a replica -If replication is not occurring, it may be necessary to reinitialize a replica. +If a replica cannot start or rejoin the cluster, or when it lags behind and can not catch up, it might be necessary to reinitialize the replica: -1. On any server in the cluster, determine the Cluster and Member names, - and check the replication lag by running `gitlab-ctl patroni members`. Here is an example: +1. [Check the replication status](#check-replication-status) to confirm which server + needs to be reinitialized. For example: ```plaintext - + Cluster: postgresql-ha (6970678148837286213) ------+---------+---------+----+-----------+ - | Member | Host | Role | State | TL | Lag in MB | - +-------------------------------------+--------------+---------+---------+----+-----------+ - | gitlab-database-1.example.com | 172.18.0.111 | Replica | running | 5 | 0 | - | gitlab-database-2.example.com | 172.18.0.112 | Replica | running | 5 | 100 | - | gitlab-database-3.example.com | 172.18.0.113 | Leader | running | 5 | | - +-------------------------------------+--------------+---------+---------+----+-----------+ + + Cluster: postgresql-ha (6970678148837286213) ------+---------+--------------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +-------------------------------------+--------------+---------+--------------+----+-----------+ + | gitlab-database-1.example.com | 172.18.0.111 | Replica | running | 55 | 0 | + | gitlab-database-2.example.com | 172.18.0.112 | Replica | start failed | | unknown | + | gitlab-database-3.example.com | 172.18.0.113 | Leader | running | 55 | | + +-------------------------------------+--------------+---------+--------------+----+-----------+ ``` -1. Reinitialize the affected replica server: +1. Sign in to the broken server and reinitialize the database and replication. Patroni will shut + down PostgreSQL on that server, remove the data directory, and reinitialize it from scratch: - ```plaintext - gitlab-ctl patroni reinitialize-replica postgresql-ha gitlab-database-2.example.com + ```shell + sudo gitlab-ctl patroni reinitialize-replica --member gitlab-database-2.example.com + ``` + + This can be run on any Patroni node, but be aware that `sudo gitlab-ctl patroni + reinitialize-replica` without `--member` will reinitialize the server it is run on. + It is recommended to run it locally on the broken server to reduce the risk of + unintended data loss. +1. Monitor the logs: + + ```shell + sudo gitlab-ctl tail patroni ``` ### Reset the Patroni state in Consul diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index 24d9792dcb0..ad4cfd11474 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -2,127 +2,14 @@ stage: Enablement group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: 'index.md' --- -# Pseudonymizer (DEPRECATED) **(ULTIMATE)** +# Pseudonymizer (removed) **(ULTIMATE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in GitLab 14.7. +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in +> GitLab 14.7 and removed in 15.0. WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in GitLab 14.7. - -Your GitLab database contains sensitive information. To protect sensitive information -when you run analytics on your database, you can use the Pseudonymizer service, which: - -1. Uses `HMAC(SHA256)` to mutate fields containing sensitive information. -1. Preserves references (referential integrity) between fields. -1. Exports your GitLab data, scrubbed of sensitive material. - -WARNING: -If the source data is available, users can compare and correlate the scrubbed data -with the original. - -To generate a pseudonymized data set: - -1. [Configure Pseudonymizer](#configure-pseudonymizer) fields and output location. -1. [Enable Pseudonymizer data collection](#enable-pseudonymizer-data-collection). -1. Optional. [Generate a data set manually](#generate-data-set-manually). - -## Configure Pseudonymizer - -To use the Pseudonymizer, configure both the fields you want to anonymize, and the location to -store the scrubbed data: - -1. **Create a manifest file**: This file describes the fields to include or pseudonymize. - - **Default manifest** - GitLab provides a default manifest in your GitLab installation - ([example `manifest.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/pseudonymizer.yml)). - To use the example manifest file, use the `config/pseudonymizer.yml` relative path - when you configure connection parameters. - - **Custom manifest** - To use a custom manifest file, use the absolute path to - the file when you configure the connection parameters. -1. **Configure connection parameters**: In the configuration method appropriate for - your version of GitLab, specify the [object storage](object_storage.md) - connection parameters (`pseudonymizer.upload.connection`). - -**For Omnibus installations:** - -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with - the values you want: - - ```ruby - gitlab_rails['pseudonymizer_manifest'] = 'config/pseudonymizer.yml' - gitlab_rails['pseudonymizer_upload_remote_directory'] = 'gitlab-elt' # bucket name - gitlab_rails['pseudonymizer_upload_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', - 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY' - } - ``` - - If you are using AWS IAM profiles, omit the AWS access key and secret access key/value pairs. - - ```ruby - gitlab_rails['pseudonymizer_upload_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'use_iam_profile' => true - } - ``` - -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - ---- - -**For installations from source:** - -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: - - ```yaml - pseudonymizer: - manifest: config/pseudonymizer.yml - upload: - remote_directory: 'gitlab-elt' # bucket name - connection: - provider: AWS - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: eu-central-1 - ``` - -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) - for the changes to take effect. - -## Enable Pseudonymizer data collection - -To enable data collection: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Metrics and Profiling**, then expand - **Pseudonymizer data collection**. -1. Select **Enable Pseudonymizer data collection**. -1. Select **Save changes**. - -## Generate data set manually - -You can also run the Pseudonymizer manually: - -1. Set these environment variables: - - `PSEUDONYMIZER_OUTPUT_DIR` - Where to store the output CSV files. Defaults to `/tmp`. - These commands produce CSV files that can be quite large. Make sure the directory - can store a file at least 10% of the size of your database. - - `PSEUDONYMIZER_BATCH` - The batch size when querying the database. Defaults to `100000`. -1. Run the command appropriate for your application: - - **Omnibus GitLab**: - `sudo gitlab-rake gitlab:db:pseudonymizer` - - **Installations from source**: - `sudo -u git -H bundle exec rake gitlab:db:pseudonymizer RAILS_ENV=production` - -After you run the command, upload the output CSV files to your configured object -storage. After the upload completes, delete the output file from the local disk. - -## Related topics - -- [Using object storage with GitLab](object_storage.md). diff --git a/doc/administration/raketasks/ldap.md b/doc/administration/raketasks/ldap.md index cae2465fad8..cdad323733d 100644 --- a/doc/administration/raketasks/ldap.md +++ b/doc/administration/raketasks/ldap.md @@ -175,7 +175,7 @@ bundle exec rake gitlab:ldap:secret:show RAILS_ENV=production ```plaintext main: password: '123' - user_bn: 'gitlab-adm' + bind_dn: 'gitlab-adm' ``` ### Edit secret diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 912cf260a03..689ed5c5824 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -79,7 +79,7 @@ In GitLab 13.0, [hashed storage](../repository_storage_types.md#hashed-storage) is enabled by default and the legacy storage is deprecated. GitLab 14.0 eliminates support for legacy storage. If you're on GitLab 13.0 and later, switching new projects to legacy storage is not possible. -The option to choose between hashed and legacy storage in the admin area has +The option to choose between hashed and legacy storage in the Admin Area has been disabled. This task must be run on any machine that has Rails/Sidekiq configured, and the task @@ -132,7 +132,7 @@ In GitLab 13.0, [hashed storage](../repository_storage_types.md#hashed-storage) is enabled by default and the legacy storage is deprecated. GitLab 14.0 eliminates support for legacy storage. If you're on GitLab 13.0 and later, switching new projects to legacy storage is not possible. -The option to choose between hashed and legacy storage in the admin area has +The option to choose between hashed and legacy storage in the Admin Area has been disabled. This task schedules all your existing projects and associated attachments to be rolled back to the @@ -169,3 +169,99 @@ Any error or warning is logged in Sidekiq's log file. If you have a Geo setup, the rollback is not reflected automatically on the **secondary** node. You may need to wait for a backfill operation to kick-in and remove the remaining repositories from the special `@hashed/` folder manually. + +## Troubleshooting + +The Rake task might not be able to complete the migration to hashed storage. +Checks on the instance will continue to report that there is legacy data: + +```plaintext +* Found 1 projects using Legacy Storage +- janedoe/testproject (id: 1234) +``` + +If you have a subscription, [raise a ticket with GitLab support](https://support.gitlab.com) +as most of the fixes are relatively high risk, involving running code on the Rails console. + +### Read only projects + +If you have [set projects read only](../troubleshooting/gitlab_rails_cheat_sheet.md#make-a-project-read-only-can-only-be-done-in-the-console) +they might fail to migrate. + +1. [Start a Rails console](../operations/rails_console.md#starting-a-rails-console-session). + +1. Check if the project is read only: + + ```ruby + project = Project.find_by_full_path('janedoe/testproject') + project.repository_read_only + ``` + +1. If it returns `true` (not `nil` or `false`), set it writable: + + ```ruby + project.update!(repository_read_only: false) + ``` + +1. [Re-run the migration Rake task](#migrate-to-hashed-storage). + +1. Set the project read-only again: + + ```ruby + project.update!(repository_read_only: true) + ``` + +### Projects pending deletion + +Check the project details in the Admin Area. If deleting the project failed +it will show as `Marked For Deletion At ..`, `Scheduled Deletion At ..` and +`pending removal`, but the dates will not be recent. + +Delete the project using the Rails console: + +1. [Start a Rails console](../operations/rails_console.md#starting-a-rails-console-session). + +1. With the following code, select the project to be deleted and account to action it: + + ```ruby + project = Project.find_by_full_path('janedoe/testproject') + user = User.find_by_username('admin_handle') + puts "\nproject selected for deletion is:\nID: #{project.id}\nPATH: #{project.full_path}\nNAME: #{project.name}\n\n" + ``` + + - Replace `janedoe/testproject` with your project path from the Rake take output or from the Admin Area. + - Replace `admin_handle` with the handle of an instance administrator or with `root`. + - Verify the output before proceeding. **There are no other checks performed**. + +1. [Destroy the project](../troubleshooting/gitlab_rails_cheat_sheet.md#destroy-a-project) **immediately**: + + ```ruby + Projects::DestroyService.new(project, user).execute + ``` + +If destroying the project generates a stack trace relating to encryption or the error `OpenSSL::Cipher::CipherError`: + +1. [Verify your GitLab secrets](check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). + +1. If the affected projects have secrets that cannot be decrypted it will be necessary to remove those specific secrets. + [Our documentation for dealing with lost secrets](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost) + is for loss of all secrets, but it's possible for specific projects to be affected. For example, + to [reset specific runner registration tokens](../../raketasks/backup_restore.md#reset-runner-registration-tokens) + for a specific project ID: + + ```sql + UPDATE projects SET runners_token = null, runners_token_encrypted = null where id = 1234; + ``` + +### `Repository cannot be moved from` errors in Sidekiq log + +Projects might fail to migrate with errors in the Sidekiq log: + +```shell +# grep 'Repository cannot be moved' /var/log/gitlab/sidekiq/current +{"severity":"ERROR","time":"2021-02-29T02:29:02.021Z","message":"Repository cannot be moved from 'janedoe/testproject' to '@hashed<value>' (PROJECT_ID=1234)"} +``` + +This might be caused by [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/259605) in the original code for hashed storage migration. + +[There is a workaround for projects still affected by this issue](https://gitlab.com/-/snippets/2039252). diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 2ca79bbeae4..f70912dbecb 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -39,7 +39,7 @@ full list of reference architectures, see <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -1351,11 +1351,18 @@ To configure the Praefect nodes, on each one: on the page. 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Praefect server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1364,7 +1371,6 @@ To configure the Praefect nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1391,8 +1397,8 @@ To configure the Praefect nodes, on each one: praefect['database_host'] = '10.6.0.141' praefect['database_port'] = 5432 # `no_proxy` settings must always be a direct connection for caching - praefect['database_host_no_proxy'] = '10.6.0.141' - praefect['database_port_no_proxy'] = 5432 + praefect['database_direct_host'] = '10.6.0.141' + praefect['database_direct_port'] = 5432 praefect['database_dbname'] = 'praefect_production' praefect['database_user'] = 'praefect' praefect['database_password'] = '<praefect_postgresql_password>' @@ -1495,10 +1501,18 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1507,7 +1521,6 @@ On each node: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1693,11 +1706,18 @@ To configure the Sidekiq nodes, on each one: on the page. 1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false gitlab_workhorse['enable'] = false prometheus['enable'] = false @@ -1705,7 +1725,6 @@ To configure the Sidekiq nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # External URL ## This should match the URL of the external load balancer @@ -2169,7 +2188,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -2202,7 +2221,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -2281,7 +2299,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -2293,7 +2311,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -2315,7 +2333,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index 7213a1eb92b..8d5afd732d9 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -62,7 +62,7 @@ monitor .[#7FFFD4,norank]--> redis @enduml ``` -The diagram above shows that while GitLab can be installed on a single server, it is internally composed of multiple services. As a GitLab instance is scaled, each of these services are broken out and independently scaled according to the demands placed on them. In some cases PaaS can be leveraged for some services (e.g. Cloud Object Storage for some file systems). For the sake of redundancy some of the services become clusters of nodes storing the same data. In a horizontal configuration of GitLab there are various ancillary services required to coordinate clusters or discover of resources (e.g. PgBouncer for Postgres connection management, Consul for Prometheus end point discovery). +The diagram above shows that while GitLab can be installed on a single server, it is internally composed of multiple services. As a GitLab instance is scaled, each of these services are broken out and independently scaled according to the demands placed on them. In some cases PaaS can be leveraged for some services (e.g. Cloud Object Storage for some file systems). For the sake of redundancy some of the services become clusters of nodes storing the same data. In a horizontal configuration of GitLab there are various ancillary services required to coordinate clusters or discover of resources (e.g. PgBouncer for PostgreSQL connection management, Consul for Prometheus end point discovery). ## Requirements diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 2a1d344508e..d9f349b59c7 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -39,7 +39,7 @@ full list of reference architectures, see <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -1355,11 +1355,18 @@ To configure the Praefect nodes, on each one: on the page. 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Praefect server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1368,7 +1375,6 @@ To configure the Praefect nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1395,8 +1401,8 @@ To configure the Praefect nodes, on each one: praefect['database_host'] = '10.6.0.141' praefect['database_port'] = 5432 # `no_proxy` settings must always be a direct connection for caching - praefect['database_host_no_proxy'] = '10.6.0.141' - praefect['database_port_no_proxy'] = 5432 + praefect['database_direct_host'] = '10.6.0.141' + praefect['database_direct_port'] = 5432 praefect['database_dbname'] = 'praefect_production' praefect['database_user'] = 'praefect' praefect['database_password'] = '<praefect_postgresql_password>' @@ -1499,10 +1505,18 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1511,7 +1525,6 @@ On each node: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1697,11 +1710,18 @@ To configure the Sidekiq nodes, on each one: on the page. 1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false gitlab_workhorse['enable'] = false prometheus['enable'] = false @@ -1709,7 +1729,6 @@ To configure the Sidekiq nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # External URL ## This should match the URL of the external load balancer @@ -2173,7 +2192,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -2206,7 +2225,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -2279,7 +2297,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -2291,7 +2309,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -2313,7 +2331,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index f417d1e2ab5..d029f356612 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -32,7 +32,7 @@ For a full list of reference architectures, see <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run as reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run as reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run as reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. <!-- markdownlint-enable MD029 --> @@ -444,21 +444,18 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: - <!-- Updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> ```ruby - # /etc/gitlab/gitlab.rb - - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' - gitlab_shell['secret_token'] = 'shellsecret' - # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -467,7 +464,6 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -486,6 +482,13 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: gitaly['listen_addr'] = "0.0.0.0:8075" gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests + # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. + # The following two values must be the same as their respective values + # of the GitLab Rails application setup + gitaly['auth_token'] = 'gitalysecret' + gitlab_shell['secret_token'] = 'shellsecret' + # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' @@ -891,7 +894,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -924,7 +927,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -1000,7 +1002,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -1012,7 +1014,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -1028,7 +1030,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. <!-- markdownlint-enable MD029 --> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 9d8be3e90b6..14b8982766c 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -48,7 +48,7 @@ For a full list of reference architectures, see <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -1295,11 +1295,18 @@ To configure the Praefect nodes, on each one: on the page. 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Praefect server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1308,7 +1315,6 @@ To configure the Praefect nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1335,8 +1341,8 @@ To configure the Praefect nodes, on each one: praefect['database_host'] = '10.6.0.141' praefect['database_port'] = 5432 # `no_proxy` settings must always be a direct connection for caching - praefect['database_host_no_proxy'] = '10.6.0.141' - praefect['database_port_no_proxy'] = 5432 + praefect['database_direct_host'] = '10.6.0.141' + praefect['database_direct_port'] = 5432 praefect['database_dbname'] = 'praefect_production' praefect['database_user'] = 'praefect' praefect['database_password'] = '<praefect_postgresql_password>' @@ -1439,10 +1445,18 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1451,7 +1465,6 @@ On each node: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1639,11 +1652,18 @@ To configure the Sidekiq nodes, one each one: on the page. 1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false gitlab_workhorse['enable'] = false prometheus['enable'] = false @@ -1651,7 +1671,6 @@ To configure the Sidekiq nodes, one each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # External URL ## This should match the URL of the external load balancer @@ -2108,7 +2127,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -2141,7 +2160,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -2239,7 +2257,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -2251,7 +2269,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -2272,7 +2290,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 6c4e2227b18..078e3a289cc 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -39,7 +39,7 @@ full list of reference architectures, see <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -86,7 +86,7 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 - + redis_cache -[hidden]-> redis_persistent } @@ -1364,11 +1364,18 @@ To configure the Praefect nodes, on each one: on the page. 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Praefect server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1377,7 +1384,6 @@ To configure the Praefect nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1404,8 +1410,8 @@ To configure the Praefect nodes, on each one: praefect['database_host'] = '10.6.0.141' praefect['database_port'] = 5432 # `no_proxy` settings must always be a direct connection for caching - praefect['database_host_no_proxy'] = '10.6.0.141' - praefect['database_port_no_proxy'] = 5432 + praefect['database_direct_host'] = '10.6.0.141' + praefect['database_direct_port'] = 5432 praefect['database_dbname'] = 'praefect_production' praefect['database_user'] = 'praefect' praefect['database_password'] = '<praefect_postgresql_password>' @@ -1508,10 +1514,18 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1520,7 +1534,6 @@ On each node: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1706,11 +1719,18 @@ To configure the Sidekiq nodes, on each one: on the page. 1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false gitlab_workhorse['enable'] = false prometheus['enable'] = false @@ -1718,7 +1738,6 @@ To configure the Sidekiq nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # External URL ## This should match the URL of the external load balancer @@ -2189,7 +2208,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -2222,7 +2241,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -2295,7 +2313,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -2304,10 +2322,10 @@ future with further specific cloud provider details. | Webservice | 16 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `m5.8xlarge` | 510 vCPU, 472 GB memory | | Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 15.5 vCPU, 50 GB memory | | Supporting services such as NGINX, Prometheus | 2 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 7.75 vCPU, 25 GB memory | - + - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -2329,7 +2347,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -2377,7 +2395,7 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 - + redis_cache -[hidden]-> redis_persistent } diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index dd7209a3a3a..d18c77902f6 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -45,7 +45,7 @@ costly-to-operate environment by using the <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. @@ -1293,11 +1293,18 @@ To configure the Praefect nodes, on each one: on the page. 1. Edit the `/etc/gitlab/gitlab.rb` file to configure Praefect: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/administration/gitaly/praefect.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Praefect server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1306,7 +1313,6 @@ To configure the Praefect nodes, on each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Praefect Configuration praefect['enable'] = true @@ -1333,8 +1339,8 @@ To configure the Praefect nodes, on each one: praefect['database_host'] = '10.6.0.141' praefect['database_port'] = 5432 # `no_proxy` settings must always be a direct connection for caching - praefect['database_host_no_proxy'] = '10.6.0.141' - praefect['database_port_no_proxy'] = 5432 + praefect['database_direct_host'] = '10.6.0.141' + praefect['database_direct_port'] = 5432 praefect['database_dbname'] = 'praefect_production' praefect['database_user'] = 'praefect' praefect['database_password'] = '<praefect_postgresql_password>' @@ -1437,10 +1443,18 @@ On each node: 1. Edit the Gitaly server node's `/etc/gitlab/gitlab.rb` file to configure storage paths, enable the network listener, and to configure the token: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/gitaly/index.md#gitaly-server-configuration +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false sidekiq['enable'] = false gitlab_workhorse['enable'] = false @@ -1449,7 +1463,6 @@ On each node: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -1635,11 +1648,18 @@ To configure the Sidekiq nodes, one each one: on the page. 1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration: +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby # Avoid running unnecessary services on the Sidekiq server gitaly['enable'] = false postgresql['enable'] = false redis['enable'] = false + nginx['enable'] = false puma['enable'] = false gitlab_workhorse['enable'] = false prometheus['enable'] = false @@ -1647,7 +1667,6 @@ To configure the Sidekiq nodes, one each one: grafana['enable'] = false gitlab_exporter['enable'] = false gitlab_kas['enable'] = false - nginx['enable'] = false # External URL ## This should match the URL of the external load balancer @@ -2108,7 +2127,7 @@ GitLab has been tested on a number of object storage providers: - [Google Cloud Storage](https://cloud.google.com/storage) - [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) -- [OpenStack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. - MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. @@ -2141,7 +2160,6 @@ on what features you intend to use: | [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | | [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | | [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) | Yes | -| [Pseudonymizer](../pseudonymizer.md) (optional feature) | No | | [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | | [Terraform state files](../terraform_state.md#using-object-storage) | Yes | @@ -2214,7 +2232,7 @@ The following tables and diagram detail the hybrid environment using the same fo as the normal environment above. First are the components that run in Kubernetes. The recommendation at this time is to -use Google Cloud's Kubernetes Engine (GKE) and associated machine types, but the memory +use Google Cloud's Kubernetes Engine (GKE) or AWS Elastic Kubernetes Service (EKS) and associated machine types, but the memory and CPU requirements should translate to most other providers. We hope to update this in the future with further specific cloud provider details. @@ -2226,7 +2244,7 @@ future with further specific cloud provider details. - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. -- Nodes configuration is shown as it is forced to ensure pod vcpu / memory ratios and avoid scaling during **performance testing**. +- Nodes configuration is shown as it is forced to ensure pod vCPU / memory ratios and avoid scaling during **performance testing**. - In production deployments, there is no need to assign pods to nodes. A minimum of three nodes in three different availability zones is strongly recommended to align with resilient cloud architecture practices. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS @@ -2247,7 +2265,7 @@ services where applicable): <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. -2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS Elasticache are known to work. +2. Can be optionally run on reputable third-party external PaaS Redis solutions. Google Memorystore and AWS ElastiCache are known to work. 3. Can be optionally run on reputable third-party load balancing services (LB PaaS). AWS ELB is known to work. 4. Should be run on reputable third-party object storage (storage PaaS) for cloud implementations. Google Cloud Storage and AWS S3 are known to work. 5. Gitaly Cluster provides the benefits of fault tolerance, but comes with additional complexity of setup and management. Review the existing [technical limitations and considerations before deploying Gitaly Cluster](../gitaly/index.md#before-deploying-gitaly-cluster). If you want sharded Gitaly, use the same specs listed above for `Gitaly`. diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index bb741c39c08..089e8881d7d 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -157,7 +157,7 @@ table.test-coverage th { </tr> <tr> <th scope="row">1k</th> - <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/1k">Daily</a> (to be moved to Weekly)</td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/1k">Weekly</a></td> <td></td> <td></td> <td></td> @@ -165,7 +165,7 @@ table.test-coverage th { </tr> <tr> <th scope="row">2k</th> - <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k">Daily</a> (to be moved to Weekly)</td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k">Weekly</a></td> <td></td> <td></td> <td></td> @@ -176,7 +176,7 @@ table.test-coverage th { <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k">Weekly</a></td> <td></td> <td></td> - <td></td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k_hybrid_aws_services">Weekly</a></td> <td></td> </tr> <tr> @@ -190,9 +190,9 @@ table.test-coverage th { <tr> <th scope="row">10k</th> <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k">Daily</a></td> - <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Past-Results/10k-Cloud-Native-Hybrid">Ad-Hoc</a></td> - <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Past-Results/10k">Ad-Hoc (inc Cloud Services)</a></td> - <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Past-Results/10k-Cloud-Native-Hybrid">Ad-Hoc</a></td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k_hybrid">Weekly</a></td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k_aws">Weekly</a></td> + <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k_hybrid_aws_services">Weekly</a></td> <td><a href="https://gitlab.com/gitlab-org/quality/performance/-/wikis/Past-Results/10k">Ad-Hoc</a></td> </tr> <tr> @@ -215,6 +215,8 @@ table.test-coverage th { ## Cost to run +The following table details the cost to run the different reference architectures across GCP, AWS, and Azure. Bare-metal costs are not included here as it varies widely depending on each customer configuration. + <table class="test-coverage"> <col> <colgroup span="2"></colgroup> @@ -356,7 +358,7 @@ Additionally, the following cloud provider services are validated and supported <tr> <td>Redis</td> <td></td> - <td>✅ <a href="https://aws.amazon.com/elasticache/" target="_blank" rel="noopener noreferrer">Elasticache</a></td> + <td>✅ <a href="https://aws.amazon.com/elasticache/" target="_blank" rel="noopener noreferrer">ElastiCache</a></td> <td></td> </tr> </tbody> diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index f33d494f638..1a593c8c6bd 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -75,7 +75,7 @@ translate between the human-readable project name and the hashed storage path. Y Administrators can look up a project's hashed path from its name or ID using: -- The [Admin area](../user/admin_area/index.md#administering-projects). +- The [Admin Area](../user/admin_area/index.md#administering-projects). - A Rails console. To look up a project's hash path in the Admin Area: @@ -154,6 +154,20 @@ WARNING: Do not run `git prune` or `git gc` in object pool repositories, which are stored in the `@pools` directory. This can cause data loss in the regular repositories that depend on the object pool. +### Group wiki storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13195) in GitLab 13.5. + +Unlike project wikis that are stored in the `@hashed` directory, group wikis are stored in a directory called `@groups`. +Like project wikis, group wikis follow the hashed storage folder convention, but use a hash of the group ID rather than the project ID. + +For example: + +```ruby +# group wiki paths +"@groups/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git" +``` + ### Object storage support This table shows which storable objects are storable in each storage type: diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md index b6c1c6a704e..d9031e2221b 100644 --- a/doc/administration/sidekiq.md +++ b/doc/administration/sidekiq.md @@ -20,21 +20,13 @@ To configure Sidekiq: 1. Edit `/etc/gitlab/gitlab.rb` with the following information and make sure to replace with your values: - ```ruby - ## - ## To maintain uniformity of links across nodes, the - ##`external_url` on the Sidekiq server should point to the external URL that users - ## use to access GitLab. This can be either: - ## - ## - The `external_url` set on your application server. - ## - The URL of a external load balancer, which routes traffic to the GitLab application server. - ## - - external_url 'https://gitlab.example.com' - - ## Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + ```ruby ######################################## ##### Services Disabled ### ######################################## @@ -45,32 +37,27 @@ To configure Sidekiq: # Enable only the services you want to run on each # specific server, while disabling all others. # - nginx['enable'] = false - grafana['enable'] = false - prometheus['enable'] = false - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false gitaly['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = false - postgres_exporter['enable'] = false postgresql['enable'] = false redis['enable'] = false - redis_exporter['enable'] = false + nginx['enable'] = false puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false gitlab_exporter['enable'] = false + gitlab_kas['enable'] = false - ####################################### - ### Sidekiq configuration ### - ####################################### - sidekiq['enable'] = true - sidekiq['listen_address'] = "0.0.0.0" - - ## Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = ['*'] * 4 - - ## Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 + ## + ## To maintain uniformity of links across nodes, the + ## `external_url` on the Sidekiq server should point to the external URL that users + ## use to access GitLab. This can be either: + ## + ## - The `external_url` set on your application server. + ## - The URL of a external load balancer, which routes traffic to the GitLab application server. + ## + external_url 'https://gitlab.example.com' ######################################## #### Redis ### @@ -89,9 +76,11 @@ To configure Sidekiq: ## Replace <gitaly_token> with the one you set up, see ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly:8075' }, + "default" => { + "gitaly_address" => "tcp://gitaly:8075", + "gitaly_token" => "<gitaly_token>" + } }) - gitlab_rails['gitaly_token'] = '<gitaly_token>' ####################################### ### Postgres ### @@ -99,16 +88,28 @@ To configure Sidekiq: # Replace <database_host> and <database_password> gitlab_rails['db_host'] = '<database_host>' - gitlab_rails['db_password'] = '<database_password>' gitlab_rails['db_port'] = '5432' - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['auto_migrate'] = false + gitlab_rails['db_password'] = '<database_password>' # Add the Sidekiq nodes to PostgreSQL's trusted addresses. # In the following example, 10.10.1.30/32 is the private IP # of the Sidekiq server. postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32) + + ## Prevent database migrations from running on upgrade automatically + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['enable'] = true + sidekiq['listen_address'] = "0.0.0.0" + + ## Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = ['*'] * 4 + + ## Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 ``` 1. Reconfigure GitLab: @@ -192,12 +193,8 @@ To configure the metrics server: ## Configure health checks -If you use health check probes to observe Sidekiq, -you can set a separate port for health checks. -Configuring health checks is only necessary if there is something that actually probes them. -For more information about health checks, see the [Sidekiq health check page](sidekiq_health_check.md). - -To enable health checks for Sidekiq: +If you use health check probes to observe Sidekiq, enable the Sidekiq health check server. +To make health checks available from `localhost:8092`: 1. Edit `/etc/gitlab/gitlab.rb`: @@ -207,16 +204,14 @@ To enable health checks for Sidekiq: sidekiq['health_checks_listen_port'] = "8092" ``` - NOTE: - If health check settings are not set, they default to the metrics exporter settings. - This default is deprecated and is set to be removed in [GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/347509). - 1. Reconfigure GitLab: ```shell sudo gitlab-ctl reconfigure ``` +For more information about health checks, see the [Sidekiq health check page](sidekiq_health_check.md). + ## Configure LDAP and user or group synchronization If you use LDAP for user and group management, you must add the LDAP configuration to your Sidekiq node as well as the LDAP diff --git a/doc/administration/sidekiq_health_check.md b/doc/administration/sidekiq_health_check.md index ae02711f46f..3ff17d3682b 100644 --- a/doc/administration/sidekiq_health_check.md +++ b/doc/administration/sidekiq_health_check.md @@ -21,8 +21,7 @@ The readiness probe checks whether the Sidekiq workers are ready to process jobs GET /readiness ``` -If you set Sidekiq's address as `localhost` and port as `8092`, -here's an example request: +If the server is bound to `localhost:8092`, the process cluster can be probed for readiness as follows: ```shell curl "http://localhost:8092/readiness" @@ -44,8 +43,7 @@ Checks whether the Sidekiq cluster is running. GET /liveness ``` -If you set Sidekiq's address as `localhost` and port as `8092`, -here's an example request: +If the server is bound to `localhost:8092`, the process cluster can be probed for liveness as follows: ```shell curl "http://localhost:8092/liveness" diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 5e8f7cb18bb..14649b82ba8 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -18,7 +18,7 @@ The storage location of these files defaults to: These locations can be configured using the options described below. -Use [external object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer-terraform-state-dependency-proxy) configuration for [GitLab Helm chart](https://docs.gitlab.com/charts/) installations. +Use [external object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-terraform-state-dependency-proxy) configuration for [GitLab Helm chart](https://docs.gitlab.com/charts/) installations. ## Disabling Terraform state @@ -144,7 +144,7 @@ You can optionally track progress and verify that all packages migrated successf Verify `objectstg` below (where `file_store=2`) has count of all states: ```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM terraform_states; +gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM terraform_state_versions; total | filesystem | objectstg ------+------------+----------- @@ -154,7 +154,7 @@ total | filesystem | objectstg Verify that there are no files on disk in the `terraform_state` folder: ```shell -sudo find /var/opt/gitlab/gitlab-rails/shared/terraform_state -type f | wc -l +sudo find /var/opt/gitlab/gitlab-rails/shared/terraform_state -type f | grep -v tmp | wc -l ``` ### S3-compatible connection settings diff --git a/doc/administration/troubleshooting/diagnostics_tools.md b/doc/administration/troubleshooting/diagnostics_tools.md index d510df5976c..479fdb963cb 100644 --- a/doc/administration/troubleshooting/diagnostics_tools.md +++ b/doc/administration/troubleshooting/diagnostics_tools.md @@ -26,4 +26,4 @@ and summarize raw `strace` data. ## kubesos -The [`kubesos`](https://gitlab.com/gitlab-com/support/toolbox/kubesos/) utiltity retrieves GitLab cluster configuration and logs from GitLab Cloud Native chart deployments. +The [`kubesos`](https://gitlab.com/gitlab-com/support/toolbox/kubesos/) utility retrieves GitLab cluster configuration and logs from GitLab Cloud Native chart deployments. diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index c45938ecd3f..97e625eb0a3 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -289,7 +289,7 @@ If the issue is: Go indexer was a beta indexer which can be optionally turned on/off, but in 12.3 it reached stable status and is now the default. - Not concerning the Go indexer, it is almost always an Elasticsearch-side issue. This means you should reach out to your Elasticsearch administrator - regarding the error(s) you are seeing. If you are unsure here, it never hurts to reach + regarding the errors you are seeing. If you are unsure here, it never hurts to reach out to GitLab support. Beyond that, review the error. If it is: diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 54d934c8986..34481582d8b 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -83,7 +83,7 @@ Notify.test_email(u.email, "Test email for #{u.name}", 'Test email').deliver_now Adding a semicolon(`;`) and a follow-up statement at the end of a statement prevents the default implicit return output. This is useful if you are already explicitly printing details and potentially have a lot of return output: ```ruby -puts ActiveRecord::Base.descendants; :ok +puts ActiveRecord::Base.descendants; :ok Project.select(&:pages_deployed?).each {|p| puts p.pages_url }; true ``` @@ -753,6 +753,21 @@ group.members_with_parents.count parent.members_with_descendants.count ``` +### Find groups that are pending deletion + +```ruby +# +# This section lists all the groups which are pending deletion +# +Group.all.each do |g| + if g.marked_for_deletion? + puts "Group ID: #{g.id}" + puts "Group name: #{g.name}" + puts "Group path: #{g.full_path}" + end +end +``` + ### Delete a group ```ruby @@ -798,6 +813,39 @@ To find features that can be toggled, run `pp p.project_feature`. Available permission levels are listed in [concerns/featurable.rb](https://gitlab.com/gitlab-org/gitlab/blob/master/app/models/concerns/featurable.rb). +### Get all error messages associated with groups, subgroups, members, and requesters + +Collect error messages associated with groups, subgroups, members, and requesters. This +captures error messages that may not appear in the Web interface. This can be especially helpful +for troubleshooting issues with [LDAP group sync](../auth/ldap/ldap_synchronization.md#group-sync) +and unexpected behavior with users and their membership in groups and subgroups. + +```ruby +# Find the group and subgroup +group = Group.find_by_full_path("parent_group") +subgroup = Group.find_by_full_path("parent_group/child_group") + +# Group and subgroup errors +group.valid? +group.errors.map(&:full_messages) + +subgroup.valid? +subgroup.errors.map(&:full_messages) + +# Group and subgroup errors for the members AND requesters +group.requesters.map(&:valid?) +group.requesters.map(&:errors).map(&:full_messages) +group.members.map(&:valid?) +group.members.map(&:errors).map(&:full_messages) +group.members_and_requesters.map(&:errors).map(&:full_messages) + +subgroup.requesters.map(&:valid?) +subgroup.requesters.map(&:errors).map(&:full_messages) +subgroup.members.map(&:valid?) +subgroup.members.map(&:errors).map(&:full_messages) +subgroup.members_and_requesters.map(&:errors).map(&:full_messages) +``` + ## Authentication ### Re-enable standard web sign-in form @@ -1191,12 +1239,6 @@ This content has been converted to a Rake task, see [verify database values can Geo::JobArtifactRegistry.failed ``` -#### Download artifact - -```ruby -Gitlab::Geo::JobArtifactDownloader.new(:job_artifact, <artifact_id>).execute -``` - #### Get a count of the synced artifacts ```ruby diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png b/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png Binary files differindex bc11e18fb6f..e002503ddc0 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png +++ b/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png b/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png Binary files differindex 78bb1725e9c..2b827e122a8 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png +++ b/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png b/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png Binary files differindex e665e23058c..8d6c5010670 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png +++ b/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md index 75c5ce460e9..bc9c39d57ea 100644 --- a/doc/administration/troubleshooting/index.md +++ b/doc/administration/troubleshooting/index.md @@ -14,6 +14,7 @@ installation. - [SSL](ssl.md) - [Geo](../geo/replication/troubleshooting.md) - [Elasticsearch](elasticsearch.md) +- [Sidekiq](sidekiq.md) - [GitLab Rails console cheat sheet](gitlab_rails_cheat_sheet.md) - [Group SAML and SCIM troubleshooting](group_saml_scim.md) **(PREMIUM SAAS)** - [Kubernetes cheat sheet](kubernetes_cheat_sheet.md) diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index c5b1d302db2..9aa490f73ef 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -12,7 +12,7 @@ but if they are not available you can still quickly parse (the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/). NOTE: -Spefically for summarising error events and basic usage statistics, +Specifically for summarizing error events and basic usage statistics, the GitLab Support Team provides the specialised [`fast-stats` tool](https://gitlab.com/gitlab-com/support/toolbox/fast-stats/#when-to-use-it). @@ -81,7 +81,7 @@ jq 'select(.status >= 500)' <FILE> #### Top 10 slowest requests ```shell -jq -s 'sort_by(-.duration) | limit(10; .[])' <FILE> +jq -s 'sort_by(-.duration_s) | limit(10; .[])' <FILE> ``` #### Find and pretty print all requests related to a project @@ -93,7 +93,7 @@ grep <PROJECT_NAME> <FILE> | jq . #### Find all requests with a total duration > 5 seconds ```shell -jq 'select(.duration > 5000)' <FILE> +jq 'select(.duration_s > 5000)' <FILE> ``` #### Find all project requests with more than 5 rugged calls @@ -105,13 +105,13 @@ grep <PROJECT_NAME> <FILE> | jq 'select(.rugged_calls > 5)' #### Find all requests with a Gitaly duration > 10 seconds ```shell -jq 'select(.gitaly_duration > 10000)' <FILE> +jq 'select(.gitaly_duration_s > 10000)' <FILE> ``` #### Find all requests with a queue duration > 10 seconds ```shell -jq 'select(.queue_duration > 10000)' <FILE> +jq 'select(.queue_duration_s > 10000)' <FILE> ``` #### Top 10 requests by # of Gitaly calls @@ -125,7 +125,7 @@ jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; #### Print the top three controller methods by request volume and their three longest durations ```shell -jq -s -r 'group_by(.controller+.action) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration) | "CT: \(length)\tMETHOD: \(.[0].controller)#\(.[0].action)\tDURS: \(.[0].duration), \(.[1].duration), \(.[2].duration)"' production_json.log +jq -s -r 'group_by(.controller+.action) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tMETHOD: \(.[0].controller)#\(.[0].action)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' production_json.log ``` **Example output** @@ -141,7 +141,7 @@ CT: 1328 METHOD: Projects::NotesController#index DURS: 403.99, 386.29, 384.3 #### Print top three routes with request count and their three longest durations ```shell -jq -s -r 'group_by(.route) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration) | "CT: \(length)\tROUTE: \(.[0].route)\tDURS: \(.[0].duration), \(.[1].duration), \(.[2].duration)"' api_json.log +jq -s -r 'group_by(.route) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tROUTE: \(.[0].route)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' api_json.log ``` **Example output** diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 62ea3bcfa3c..7a64bcc9b87 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -315,7 +315,7 @@ queue.each { |job| job.delete if <condition>} Have a look at the section below for cancelling running jobs. -In the method above, `<queue-name>` is the name of the queue that contains the job(s) you want to delete and `<condition>` decides which jobs get deleted. +In the method above, `<queue-name>` is the name of the queue that contains the jobs you want to delete and `<condition>` decides which jobs get deleted. Commonly, `<condition>` references the job arguments, which depend on the type of job in question. To find the arguments for a specific queue, you can have a look at the `perform` function of the related worker file, commonly found at `/app/workers/<queue-name>_worker.rb`. diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md index 5fe493a536c..0821f952d61 100644 --- a/doc/administration/troubleshooting/test_environments.md +++ b/doc/administration/troubleshooting/test_environments.md @@ -52,6 +52,8 @@ gitlab/gitlab-ee:11.5.3-ee.0 #### SAML for Authentication +In the following examples, when replacing `<GITLAB_IP_OR_DOMAIN>` and `<SAML_IP_OR_DOMAIN>` it is important to prepend your IP or domain name, with the protocol (`http://` or `https://`) being used. + We can use the [`test-saml-idp` Docker image](https://hub.docker.com/r/jamedjo/test-saml-idp) to do the work for us: diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index aba7b5fc318..d66aa5945b6 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -67,8 +67,6 @@ For source installations the following settings are nested under `uploads:` and |---------|-------------|---------| | `enabled` | Enable/disable object storage | `false` | | `remote_directory` | The bucket name where Uploads will be stored| | -| `direct_upload` | Set to `true` to remove Puma from the Upload path. Workhorse handles the actual Artifact Upload to Object Storage while Puma does minimal processing to keep track of the upload. There is no need for local shared storage. The option may be removed if support for a single storage type for all files is introduced. Read more on [direct upload](../development/uploads/implementation.md#direct-upload). | `false` | -| `background_upload` | Set to `false` to disable automatic upload. Option may be removed once upload is direct to S3 (if `direct_upload` is set to `true` it will override `background_upload`) | `true` | | `proxy_download` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` | | `connection` | Various connection options described below | | @@ -130,60 +128,3 @@ _The uploads are stored by default in 1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). - -#### OpenStack example - -**In Omnibus installations:** - -_The uploads are stored by default in -`/var/opt/gitlab/gitlab-rails/uploads`._ - -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with - the values you want: - - ```ruby - gitlab_rails['uploads_object_store_remote_directory'] = "OPENSTACK_OBJECT_CONTAINER_NAME" - gitlab_rails['uploads_object_store_connection'] = { - 'provider' => 'OpenStack', - 'openstack_username' => 'OPENSTACK_USERNAME', - 'openstack_api_key' => 'OPENSTACK_PASSWORD', - 'openstack_temp_url_key' => 'OPENSTACK_TEMP_URL_KEY', - 'openstack_auth_url' => 'https://auth.cloud.ovh.net/v2.0/', - 'openstack_region' => 'DE1', - 'openstack_tenant' => 'TENANT_ID', - } - ``` - -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). - ---- - -**In installations from source:** - -_The uploads are stored by default in -`/home/git/gitlab/public/uploads`._ - -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: - - ```yaml - uploads: - object_store: - enabled: true - direct_upload: false - background_upload: true - proxy_download: false - remote_directory: OPENSTACK_OBJECT_CONTAINER_NAME - connection: - provider: OpenStack - openstack_username: OPENSTACK_USERNAME - openstack_api_key: OPENSTACK_PASSWORD - openstack_temp_url_key: OPENSTACK_TEMP_URL_KEY - openstack_auth_url: 'https://auth.cloud.ovh.net/v2.0/' - openstack_region: DE1 - openstack_tenant: 'TENANT_ID' - ``` - -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate:all` Rake task](raketasks/uploads/migrate.md). diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index f6d1e554aae..9408e7c25a6 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -24,7 +24,7 @@ The following API resources are available in the project context: | Resource | Available endpoints | |:------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Access requests](access_requests.md) | `/projects/:id/access_requests` (also available for groups) | -| [Access tokens](resource_access_tokens.md) | `/projects/:id/access_tokens` (also available for groups) | +| [Access tokens](project_access_tokens.md) | `/projects/:id/access_tokens` (also available for groups) | | [Agents](cluster_agents.md) | `/projects/:id/cluster_agents` | | [Award emoji](award_emoji.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` | | [Branches](branches.md) | `/projects/:id/repository/branches/`, `/projects/:id/repository/merged_branches` | @@ -70,7 +70,7 @@ The following API resources are available in the project context: | [Project milestones](milestones.md) | `/projects/:id/milestones` | | [Project snippets](project_snippets.md) | `/projects/:id/snippets` | | [Project templates](project_templates.md) | `/projects/:id/templates` | -| [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE)** | `/projects/:id/templates` | +| [Project vulnerabilities](project_vulnerabilities.md) **(ULTIMATE)** | `/projects/:id/vulnerabilities` | | [Project wikis](wikis.md) | `/projects/:id/wikis` | | [Project-level variables](project_level_variables.md) | `/projects/:id/variables` | | [Projects](projects.md) including setting Webhooks | `/projects`, `/projects/:id/hooks` (also available for users) | diff --git a/doc/api/cluster_agents.md b/doc/api/cluster_agents.md index 37cc4a24342..1c2a3b52761 100644 --- a/doc/api/cluster_agents.md +++ b/doc/api/cluster_agents.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Agents API **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83270) in GitLab 14.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83270) in GitLab 14.10. +> - Agent Tokens API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. Use the Agents API to work with the GitLab agent for Kubernetes. @@ -236,3 +237,216 @@ Example request: ```shell curl --request DELETE --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/20/cluster_agents/1 ``` + +## List tokens for an agent + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. + +Returns a list of tokens for an agent. + +You must have at least the Developer role to use this endpoint. + +```plaintext +GET /projects/:id/cluster_agents/:agent_id/tokens +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|------------|-------------------|-----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) maintained by the authenticated user. | +| `agent_id` | integer or string | yes | ID of the agent. | + +Response: + +The response is a list of tokens with the following fields: + +| Attribute | Type | Description | +|----------------------|----------------|-------------------------------------------------------------------| +| `id` | integer | ID of the token. | +| `name` | string | Name of the token. | +| `description` | string or null | Description of the token. | +| `agent_id` | integer | ID of the agent the token belongs to. | +| `status` | string | The status of the token. Valid values are `active` and `revoked`. | +| `created_at` | string | ISO8601 datetime when the token was created. | +| `created_by_user_id` | string | User ID of the user who created the token. | + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/20/cluster_agents/5/tokens" +``` + +Example response: + +```json +[ + { + "id": 1, + "name": "abcd", + "description": "Some token", + "agent_id": 5, + "status": "active", + "created_at": "2022-03-25T14:12:11.497Z", + "created_by_user_id": 1 + }, + { + "id": 2, + "name": "foobar", + "description": null, + "agent_id": 5, + "status": "active", + "created_at": "2022-03-25T14:12:11.497Z", + "created_by_user_id": 1 + } +] +``` + +NOTE: +The `last_used_at` field for a token is only returned when getting a single agent token. + +## Get a single agent token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. + +Gets a single agent token. + +You must have at least the Developer role to use this endpoint. + +```shell +GET /projects/:id/cluster_agents/:agent_id/tokens/:token_id +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|------------|-------------------|----------|-------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) maintained by the authenticated user. | +| `agent_id` | integer | yes | ID of the agent. | +| `token_id` | integer | yes | ID of the token. | + +Response: + +The response is a single token with the following fields: + +| Attribute | Type | Description | +|----------------------|----------------|-------------------------------------------------------------------| +| `id` | integer | ID of the token. | +| `name` | string | Name of the token. | +| `description` | string or null | Description of the token. | +| `agent_id` | integer | ID of the agent the token belongs to. | +| `status` | string | The status of the token. Valid values are `active` and `revoked`. | +| `created_at` | string | ISO8601 datetime when the token was created. | +| `created_by_user_id` | string | User ID of the user who created the token. | +| `last_used_at` | string or null | ISO8601 datetime when the token was last used. | + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/20/cluster_agents/5/token/1" +``` + +Example response: + +```json +{ + "id": 1, + "name": "abcd", + "description": "Some token", + "agent_id": 5, + "status": "active", + "created_at": "2022-03-25T14:12:11.497Z", + "created_by_user_id": 1, + "last_used_at": null +} +``` + +## Create an agent token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. + +Creates a new token for an agent. + +You must have at least the Maintainer role to use this endpoint. + +```shell +POST /projects/:id/cluster_agents/:agent_id/tokens +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|---------------|-------------------|----------|------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) maintained by the authenticated user. | +| `agent_id` | integer | yes | ID of the agent. | +| `name` | string | yes | Name for the token. | +| `description` | string | no | Description for the token. | + +Response: + +The response is the new token with the following fields: + +| Attribute | Type | Description | +|----------------------|----------------|-------------------------------------------------------------------| +| `id` | integer | ID of the token. | +| `name` | string | Name of the token. | +| `description` | string or null | Description of the token. | +| `agent_id` | integer | ID of the agent the token belongs to. | +| `status` | string | The status of the token. Valid values are `active` and `revoked`. | +| `created_at` | string | ISO8601 datetime when the token was created. | +| `created_by_user_id` | string | User ID of the user who created the token. | +| `last_used_at` | string or null | ISO8601 datetime when the token was last used. | +| `token` | string | The secret token value. | + +NOTE: +The `token` is only returned in the response of the `POST` endpoint and cannot be retrieved afterwards. + +Example request: + +```shell +curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/20/cluster_agents/5/tokens" \ + -H "Content-Type:application/json" \ + -X POST --data '{"name":"some-token"}' +``` + +Example response: + +```json +{ + "id": 1, + "name": "abcd", + "description": "Some token", + "agent_id": 5, + "status": "active", + "created_at": "2022-03-25T14:12:11.497Z", + "created_by_user_id": 1, + "last_used_at": null, + "token": "qeY8UVRisx9y3Loxo1scLxFuRxYcgeX3sxsdrpP_fR3Loq4xyg" +} +``` + +## Revoke an agent token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. + +Revokes an agent token. + +You must have at least the Maintainer role to use this endpoint. + +```plaintext +DELETE /projects/:id/cluster_agents/:agent_id/tokens/:token_id +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|------------|-------------------|----------|---------------------------------------------------------------------------------------------------------------- -| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) maintained by the authenticated user. | +| `agent_id` | integer | yes | ID of the agent. | +| `token_id` | integer | yes | ID of the token. | + +Example request: + +```shell +curl --request DELETE --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/20/cluster_agents/5/tokens/1 +``` diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index eb5a124c40c..0dd2106b4d1 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -127,6 +127,8 @@ Example response: ### Within a group +> [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/336912) the `tags` and `tag_count` attributes in GitLab 15.0. + Get a list of registry repositories in a group. ```plaintext @@ -136,12 +138,10 @@ GET /groups/:id/registry/repositories | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) accessible by the authenticated user. | -| `tags` | boolean | no | If the parameter is included as true, each repository includes an array of `"tags"` in the response. | -| `tags_count` | boolean | no | If the parameter is included as true, each repository includes `"tags_count"` in the response ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32141) in GitLab 13.1). | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" \ - "https://gitlab.example.com/api/v4/groups/2/registry/repositories?tags=1&tags_count=true" + "https://gitlab.example.com/api/v4/groups/2/registry/repositories" ``` Example response: @@ -156,14 +156,6 @@ Example response: "location": "gitlab.example.com:5000/group/project", "created_at": "2019-01-10T13:38:57.391Z", "cleanup_policy_started_at": "2020-08-17T03:12:35.489Z", - "tags_count": 1, - "tags": [ - { - "name": "0.0.1", - "path": "group/project:0.0.1", - "location": "gitlab.example.com:5000/group/project:0.0.1" - } - ] }, { "id": 2, @@ -173,24 +165,6 @@ Example response: "location": "gitlab.example.com:5000/group/other_project", "created_at": "2019-01-10T13:39:08.229Z", "cleanup_policy_started_at": "2020-01-10T15:40:57.391Z", - "tags_count": 3, - "tags": [ - { - "name": "0.0.1", - "path": "group/other_project:0.0.1", - "location": "gitlab.example.com:5000/group/other_project:0.0.1" - }, - { - "name": "0.0.2", - "path": "group/other_project:0.0.2", - "location": "gitlab.example.com:5000/group/other_project:0.0.2" - }, - { - "name": "latest", - "path": "group/other_project:latest", - "location": "gitlab.example.com:5000/group/other_project:latest" - } - ] } ] ``` @@ -446,7 +420,7 @@ These are different from project or personal access tokens in the GitLab applica GET /v2/_catalog ``` -To list all container repositories on your GitLab instance, admin credentials are required: +To list all container repositories on your GitLab instance, administrator credentials are required: ```shell $ curl --request GET --user "<admin-username>:<admin-password>" "https://gitlab.example.com/jwt/auth?service=container_registry&scope=registry:catalog:*" diff --git a/doc/api/dora4_project_analytics.md b/doc/api/dora4_project_analytics.md index c202b075c42..53170c3a77f 100644 --- a/doc/api/dora4_project_analytics.md +++ b/doc/api/dora4_project_analytics.md @@ -10,7 +10,7 @@ type: reference, api > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/279039) in GitLab 13.7. WARNING: -These endpoints are deprecated and will be removed in GitLab 14.0. Use the [DORA metrics API](dora/metrics.md) instead. +These endpoints have been removed in GitLab 14.0. Use the [DORA metrics API](dora/metrics.md) instead. All methods require reporter authorization. diff --git a/doc/api/environments.md b/doc/api/environments.md index 40d161485ff..bfc097d44ee 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -36,6 +36,7 @@ Example response: "slug": "review-fix-foo-dfjre3", "external_url": "https://review-fix-foo-dfjre3.gitlab.example.com", "state": "available", + "tier": "development", "created_at": "2019-05-25T18:55:13.252Z", "updated_at": "2019-05-27T18:55:13.252Z", "enable_advanced_logs_querying": false, @@ -147,6 +148,7 @@ Example of response "slug": "review-fix-foo-dfjre3", "external_url": "https://review-fix-foo-dfjre3.gitlab.example.com", "state": "available", + "tier": "development", "created_at": "2019-05-25T18:55:13.252Z", "updated_at": "2019-05-27T18:55:13.252Z", "enable_advanced_logs_querying": false, @@ -249,6 +251,7 @@ POST /projects/:id/environments | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | yes | The name of the environment | | `external_url` | string | no | Place to link to for this environment | +| `tier` | string | no | The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other` | ```shell curl --data "name=deploy&external_url=https://deploy.gitlab.example.com" \ @@ -264,6 +267,7 @@ Example response: "slug": "deploy", "external_url": "https://deploy.gitlab.example.com", "state": "available", + "tier": "production", "created_at": "2019-05-25T18:55:13.252Z", "updated_at": "2019-05-27T18:55:13.252Z" } @@ -285,6 +289,7 @@ PUT /projects/:id/environments/:environments_id | `environment_id` | integer | yes | The ID of the environment | | `name` | string | no | [Deprecated and will be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/338897) | | `external_url` | string | no | The new `external_url` | +| `tier` | string | no | The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other` | ```shell curl --request PUT --data "name=staging&external_url=https://staging.gitlab.example.com" \ @@ -300,6 +305,7 @@ Example response: "slug": "staging", "external_url": "https://staging.gitlab.example.com", "state": "available", + "tier": "staging", "created_at": "2019-05-25T18:55:13.252Z", "updated_at": "2019-05-27T18:55:13.252Z" } diff --git a/doc/api/features.md b/doc/api/features.md index 0ad76829651..346f4879358 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -129,11 +129,12 @@ POST /features/:name | `feature_group` | string | no | A Feature group name | | `user` | string | no | A GitLab username | | `group` | string | no | A GitLab group's path, for example `gitlab-org` | +| `namespace` | string | no | A GitLab group or user namespace's path, for example `gitlab-org` or username path. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353117) in GitLab 15.0. | | `project` | string | no | A projects path, for example `gitlab-org/gitlab-foss` | | `force` | boolean | no | Skip feature flag validation checks, such as a YAML definition | You can enable or disable a feature for a `feature_group`, a `user`, -a `group`, and a `project` in a single API call. +a `group`, a `namespace` and a `project` in a single API call. ```shell curl --data "value=30" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/features/new_library" diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 457b083c217..2ad29e8cb45 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GraphQL API **(FREE)** -> [Generally available](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30444) in GitLab 12.1. [Feature flag `graphql`](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30444) removed. +> Enabled and [made generally available](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30444) in GitLab 12.1. [Feature flag `graphql`](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30444) removed. [GraphQL](https://graphql.org/) is a query language for APIs. You can use it to request the exact data you need, and therefore limit the number of requests you need. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 7ef9897a170..97ef81b8bfa 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -181,29 +181,6 @@ Fields related to Instance Security Dashboard. Returns [`InstanceSecurityDashboard`](#instancesecuritydashboard). -### `Query.instanceStatisticsMeasurements` - -Get statistics on the instance. - -WARNING: -**Deprecated** in 13.10. -This was renamed. -Use: [`Query.usageTrendsMeasurements`](#queryusagetrendsmeasurements). - -Returns [`UsageTrendsMeasurementConnection`](#usagetrendsmeasurementconnection). - -This field returns a [connection](#connections). It accepts the -four standard [pagination arguments](#connection-pagination-arguments): -`before: String`, `after: String`, `first: Int`, `last: Int`. - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="queryinstancestatisticsmeasurementsidentifier"></a>`identifier` | [`MeasurementIdentifier!`](#measurementidentifier) | Type of measurement or statistics to retrieve. | -| <a id="queryinstancestatisticsmeasurementsrecordedafter"></a>`recordedAfter` | [`Time`](#time) | Measurement recorded after this date. | -| <a id="queryinstancestatisticsmeasurementsrecordedbefore"></a>`recordedBefore` | [`Time`](#time) | Measurement recorded before this date. | - ### `Query.issue` Find an issue. @@ -624,6 +601,7 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="mutationadminsidekiqqueuesdeletejobsartifactsize"></a>`artifactSize` | [`String`](#string) | Delete jobs matching artifact_size in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobscallerid"></a>`callerId` | [`String`](#string) | Delete jobs matching caller_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsclientid"></a>`clientId` | [`String`](#string) | Delete jobs matching client_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | @@ -872,6 +850,11 @@ Input type: `BulkEnableDevopsAdoptionNamespacesInput` ### `Mutation.ciCdSettingsUpdate` +WARNING: +**Deprecated** in 15.0. +This was renamed. +Use: `ProjectCiCdSettingsUpdate`. + Input type: `CiCdSettingsUpdateInput` #### Arguments @@ -973,28 +956,6 @@ Input type: `ClusterAgentTokenCreateInput` | <a id="mutationclusteragenttokencreatesecret"></a>`secret` | [`String`](#string) | Token secret value. Make sure you save it - you won't be able to access it again. | | <a id="mutationclusteragenttokencreatetoken"></a>`token` | [`ClusterAgentToken`](#clusteragenttoken) | Token created after mutation. | -### `Mutation.clusterAgentTokenDelete` - -WARNING: -**Deprecated** in 14.7. -Tokens must be revoked with ClusterAgentTokenRevoke. - -Input type: `ClusterAgentTokenDeleteInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationclusteragenttokendeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationclusteragenttokendeleteid"></a>`id` | [`ClustersAgentTokenID!`](#clustersagenttokenid) | Global ID of the cluster agent token that will be deleted. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationclusteragenttokendeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationclusteragenttokendeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | - ### `Mutation.clusterAgentTokenRevoke` Input type: `ClusterAgentTokenRevokeInput` @@ -1372,7 +1333,7 @@ Input type: `CreateEpicInput` | ---- | ---- | ----------- | | <a id="mutationcreateepicaddlabelids"></a>`addLabelIds` | [`[ID!]`](#id) | IDs of labels to be added to the epic. | | <a id="mutationcreateepicclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationcreateepiccolor"></a>`color` | [`String`](#string) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | +| <a id="mutationcreateepiccolor"></a>`color` | [`Color`](#color) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | | <a id="mutationcreateepicconfidential"></a>`confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | | <a id="mutationcreateepicdescription"></a>`description` | [`String`](#string) | Description of the epic. | | <a id="mutationcreateepicduedatefixed"></a>`dueDateFixed` | [`String`](#string) | End date of the epic. | @@ -1611,6 +1572,7 @@ Input type: `CustomerRelationsContactUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="mutationcustomerrelationscontactupdateactive"></a>`active` | [`Boolean`](#boolean) | State of the contact. | | <a id="mutationcustomerrelationscontactupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationcustomerrelationscontactupdatedescription"></a>`description` | [`String`](#string) | Description of or notes for the contact. | | <a id="mutationcustomerrelationscontactupdateemail"></a>`email` | [`String`](#string) | Email address of the contact. | @@ -1658,6 +1620,7 @@ Input type: `CustomerRelationsOrganizationUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="mutationcustomerrelationsorganizationupdateactive"></a>`active` | [`Boolean`](#boolean) | State of the organization. | | <a id="mutationcustomerrelationsorganizationupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationcustomerrelationsorganizationupdatedefaultrate"></a>`defaultRate` | [`Float`](#float) | Standard billing rate for the organization. | | <a id="mutationcustomerrelationsorganizationupdatedescription"></a>`description` | [`String`](#string) | Description of or notes for the organization. | @@ -2708,7 +2671,7 @@ Input type: `ExternalAuditEventDestinationUpdateInput` | ---- | ---- | ----------- | | <a id="mutationexternalauditeventdestinationupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationexternalauditeventdestinationupdatedestinationurl"></a>`destinationUrl` | [`String`](#string) | Destination URL to change. | -| <a id="mutationexternalauditeventdestinationupdateid"></a>`id` | [`AuditEventsExternalAuditEventDestinationID!`](#auditeventsexternalauditeventdestinationid) | ID of external audit event destination to destroy. | +| <a id="mutationexternalauditeventdestinationupdateid"></a>`id` | [`AuditEventsExternalAuditEventDestinationID!`](#auditeventsexternalauditeventdestinationid) | ID of external audit event destination to update. | #### Fields @@ -3151,12 +3114,12 @@ Input type: `IterationCadenceCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationiterationcadencecreateactive"></a>`active` | [`Boolean!`](#boolean) | Whether the iteration cadence is active. | -| <a id="mutationiterationcadencecreateautomatic"></a>`automatic` | [`Boolean!`](#boolean) | Whether the iteration cadence should automatically generate future iterations. | +| <a id="mutationiterationcadencecreateautomatic"></a>`automatic` | [`Boolean!`](#boolean) | Whether the iteration cadence should automatically generate upcoming iterations. | | <a id="mutationiterationcadencecreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationiterationcadencecreatedescription"></a>`description` | [`String`](#string) | Description of the iteration cadence. Maximum length is 5000 characters. | | <a id="mutationiterationcadencecreatedurationinweeks"></a>`durationInWeeks` | [`Int`](#int) | Duration in weeks of the iterations within this cadence. | | <a id="mutationiterationcadencecreategrouppath"></a>`groupPath` | [`ID!`](#id) | Group where the iteration cadence is created. | -| <a id="mutationiterationcadencecreateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Future iterations to be created when iteration cadence is set to automatic. | +| <a id="mutationiterationcadencecreateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="mutationiterationcadencecreaterollover"></a>`rollOver` | [`Boolean`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | | <a id="mutationiterationcadencecreatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | | <a id="mutationiterationcadencecreatetitle"></a>`title` | [`String`](#string) | Title of the iteration cadence. | @@ -3197,12 +3160,12 @@ Input type: `IterationCadenceUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationiterationcadenceupdateactive"></a>`active` | [`Boolean`](#boolean) | Whether the iteration cadence is active. | -| <a id="mutationiterationcadenceupdateautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate future iterations. | +| <a id="mutationiterationcadenceupdateautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate upcoming iterations. | | <a id="mutationiterationcadenceupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationiterationcadenceupdatedescription"></a>`description` | [`String`](#string) | Description of the iteration cadence. Maximum length is 5000 characters. | | <a id="mutationiterationcadenceupdatedurationinweeks"></a>`durationInWeeks` | [`Int`](#int) | Duration in weeks of the iterations within this cadence. | | <a id="mutationiterationcadenceupdateid"></a>`id` | [`IterationsCadenceID!`](#iterationscadenceid) | Global ID of the iteration cadence. | -| <a id="mutationiterationcadenceupdateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Future iterations to be created when iteration cadence is set to automatic. | +| <a id="mutationiterationcadenceupdateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="mutationiterationcadenceupdaterollover"></a>`rollOver` | [`Boolean`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | | <a id="mutationiterationcadenceupdatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | | <a id="mutationiterationcadenceupdatetitle"></a>`title` | [`String`](#string) | Title of the iteration cadence. | @@ -3481,6 +3444,48 @@ Input type: `MergeRequestCreateInput` | <a id="mutationmergerequestcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationmergerequestcreatemergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | +### `Mutation.mergeRequestRemoveAttentionRequest` + +Input type: `MergeRequestRemoveAttentionRequestInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationmergerequestremoveattentionrequestclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestremoveattentionrequestiid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | +| <a id="mutationmergerequestremoveattentionrequestprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| <a id="mutationmergerequestremoveattentionrequestuserid"></a>`userId` | [`UserID!`](#userid) | User ID of the user for attention request removal. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationmergerequestremoveattentionrequestclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestremoveattentionrequesterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationmergerequestremoveattentionrequestmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | + +### `Mutation.mergeRequestRequestAttention` + +Input type: `MergeRequestRequestAttentionInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationmergerequestrequestattentionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestrequestattentioniid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | +| <a id="mutationmergerequestrequestattentionprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| <a id="mutationmergerequestrequestattentionuserid"></a>`userId` | [`UserID!`](#userid) | User ID of the user to request attention. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationmergerequestrequestattentionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestrequestattentionerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationmergerequestrequestattentionmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | + ### `Mutation.mergeRequestReviewerRereview` Input type: `MergeRequestReviewerRereviewInput` @@ -3632,8 +3637,6 @@ Input type: `MergeRequestSetSubscriptionInput` ### `Mutation.mergeRequestToggleAttentionRequested` -Available only when feature flag `mr_attention_requests` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. - Input type: `MergeRequestToggleAttentionRequestedInput` #### Arguments @@ -3679,6 +3682,26 @@ Input type: `MergeRequestUpdateInput` | <a id="mutationmergerequestupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationmergerequestupdatemergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | +### `Mutation.namespaceCiCdSettingsUpdate` + +Input type: `NamespaceCiCdSettingsUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationnamespacecicdsettingsupdateallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean`](#boolean) | Indicates if stale runners directly belonging to this namespace should be periodically pruned. | +| <a id="mutationnamespacecicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationnamespacecicdsettingsupdatefullpath"></a>`fullPath` | [`ID!`](#id) | Full path of the namespace the settings belong to. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationnamespacecicdsettingsupdatecicdsettings"></a>`ciCdSettings` | [`NamespaceCiCdSetting!`](#namespacecicdsetting) | CI/CD settings after mutation. | +| <a id="mutationnamespacecicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationnamespacecicdsettingsupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.namespaceIncreaseStorageTemporarily` Input type: `NamespaceIncreaseStorageTemporarilyInput` @@ -3890,6 +3913,29 @@ Input type: `PipelineRetryInput` | <a id="mutationpipelineretryerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationpipelineretrypipeline"></a>`pipeline` | [`Pipeline`](#pipeline) | Pipeline after mutation. | +### `Mutation.projectCiCdSettingsUpdate` + +Input type: `ProjectCiCdSettingsUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationprojectcicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationprojectcicdsettingsupdatefullpath"></a>`fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | +| <a id="mutationprojectcicdsettingsupdatejobtokenscopeenabled"></a>`jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI job tokens generated in this project have restricted access to resources. | +| <a id="mutationprojectcicdsettingsupdatekeeplatestartifact"></a>`keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for this project. | +| <a id="mutationprojectcicdsettingsupdatemergepipelinesenabled"></a>`mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | +| <a id="mutationprojectcicdsettingsupdatemergetrainsenabled"></a>`mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationprojectcicdsettingsupdatecicdsettings"></a>`ciCdSettings` | [`ProjectCiCdSetting!`](#projectcicdsetting) | CI/CD settings after mutation. | +| <a id="mutationprojectcicdsettingsupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationprojectcicdsettingsupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.projectSetComplianceFramework` Assign (or unset) a compliance framework to a project. @@ -4355,7 +4401,7 @@ Input type: `SecurityPolicyProjectAssignInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationsecuritypolicyprojectassignclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationsecuritypolicyprojectassignfullpath"></a>`fullPath` | [`String`](#string) | Full path of the project. | +| <a id="mutationsecuritypolicyprojectassignfullpath"></a>`fullPath` | [`String`](#string) | Full path of the project or group. | | <a id="mutationsecuritypolicyprojectassignprojectpath"></a>`projectPath` **{warning-solid}** | [`ID`](#id) | **Deprecated:** Use `fullPath`. Deprecated in 14.10. | | <a id="mutationsecuritypolicyprojectassignsecuritypolicyprojectid"></a>`securityPolicyProjectId` | [`ProjectID!`](#projectid) | ID of the security policy project. | @@ -4377,7 +4423,7 @@ Input type: `SecurityPolicyProjectCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationsecuritypolicyprojectcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationsecuritypolicyprojectcreatefullpath"></a>`fullPath` | [`String`](#string) | Full path of the project. | +| <a id="mutationsecuritypolicyprojectcreatefullpath"></a>`fullPath` | [`String`](#string) | Full path of the project or group. | | <a id="mutationsecuritypolicyprojectcreateprojectpath"></a>`projectPath` **{warning-solid}** | [`ID`](#id) | **Deprecated:** Use `fullPath`. Deprecated in 14.10. | #### Fields @@ -4399,7 +4445,7 @@ Input type: `SecurityPolicyProjectUnassignInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationsecuritypolicyprojectunassignclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationsecuritypolicyprojectunassignfullpath"></a>`fullPath` | [`String`](#string) | Full path of the project. | +| <a id="mutationsecuritypolicyprojectunassignfullpath"></a>`fullPath` | [`String`](#string) | Full path of the project or group. | | <a id="mutationsecuritypolicyprojectunassignprojectpath"></a>`projectPath` **{warning-solid}** | [`ID`](#id) | **Deprecated:** Use `fullPath`. Deprecated in 14.10. | #### Fields @@ -4565,6 +4611,25 @@ Input type: `TimelineEventUpdateInput` | <a id="mutationtimelineeventupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationtimelineeventupdatetimelineevent"></a>`timelineEvent` | [`TimelineEventType`](#timelineeventtype) | Timeline event. | +### `Mutation.timelogDelete` + +Input type: `TimelogDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationtimelogdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationtimelogdeleteid"></a>`id` | [`TimelogID!`](#timelogid) | Global ID of the timelog. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationtimelogdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationtimelogdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationtimelogdeletetimelog"></a>`timelog` | [`Timelog`](#timelog) | Deleted timelog. | + ### `Mutation.todoCreate` Input type: `TodoCreateInput` @@ -4850,7 +4915,7 @@ Input type: `UpdateEpicInput` | ---- | ---- | ----------- | | <a id="mutationupdateepicaddlabelids"></a>`addLabelIds` | [`[ID!]`](#id) | IDs of labels to be added to the epic. | | <a id="mutationupdateepicclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationupdateepiccolor"></a>`color` | [`String`](#string) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | +| <a id="mutationupdateepiccolor"></a>`color` | [`Color`](#color) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | | <a id="mutationupdateepicconfidential"></a>`confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | | <a id="mutationupdateepicdescription"></a>`description` | [`String`](#string) | Description of the epic. | | <a id="mutationupdateepicduedatefixed"></a>`dueDateFixed` | [`String`](#string) | End date of the epic. | @@ -5348,6 +5413,29 @@ Input type: `WorkItemDeleteInput` | <a id="mutationworkitemdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationworkitemdeleteproject"></a>`project` | [`Project`](#project) | Project the deleted work item belonged to. | +### `Mutation.workItemDeleteTask` + +Deletes a task in a work item's description. Available only when feature flag `work_items` is enabled. This feature is experimental and is subject to change without notice. + +Input type: `WorkItemDeleteTaskInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemdeletetaskclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemdeletetaskid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| <a id="mutationworkitemdeletetasklockversion"></a>`lockVersion` | [`Int!`](#int) | Current lock version of the work item containing the task in the description. | +| <a id="mutationworkitemdeletetasktaskdata"></a>`taskData` | [`WorkItemDeletedTaskInput!`](#workitemdeletedtaskinput) | Arguments necessary to delete a task from a work item's description. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemdeletetaskclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemdeletetaskerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationworkitemdeletetaskworkitem"></a>`workItem` | [`WorkItem`](#workitem) | Updated work item. | + ### `Mutation.workItemUpdate` Updates a work item by Global ID. Available only when feature flag `work_items` is enabled. The feature is experimental and is subject to change without notice. @@ -7358,6 +7446,30 @@ The edge type for [`OncallParticipantType`](#oncallparticipanttype). | <a id="oncallparticipanttypeedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="oncallparticipanttypeedgenode"></a>`node` | [`OncallParticipantType`](#oncallparticipanttype) | The item at the end of the edge. | +#### `PackageBaseConnection` + +The connection type for [`PackageBase`](#packagebase). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="packagebaseconnectioncount"></a>`count` | [`Int!`](#int) | Total count of collection. | +| <a id="packagebaseconnectionedges"></a>`edges` | [`[PackageBaseEdge]`](#packagebaseedge) | A list of edges. | +| <a id="packagebaseconnectionnodes"></a>`nodes` | [`[PackageBase]`](#packagebase) | A list of nodes. | +| <a id="packagebaseconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `PackageBaseEdge` + +The edge type for [`PackageBase`](#packagebase). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="packagebaseedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="packagebaseedgenode"></a>`node` | [`PackageBase`](#packagebase) | The item at the end of the edge. | + #### `PackageConnection` The connection type for [`Package`](#package). @@ -9266,6 +9378,7 @@ Represents the total number of issues and their weights for a particular day. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="ciconfigerrors"></a>`errors` | [`[String!]`](#string) | Linting errors. | +| <a id="ciconfigincludes"></a>`includes` | [`[CiConfigInclude!]`](#ciconfiginclude) | List of included files. | | <a id="ciconfigmergedyaml"></a>`mergedYaml` | [`String`](#string) | Merged CI configuration YAML. | | <a id="ciconfigstages"></a>`stages` | [`CiConfigStageConnection`](#ciconfigstageconnection) | Stages of the pipeline. (see [Connections](#connections)) | | <a id="ciconfigstatus"></a>`status` | [`CiConfigStatus`](#ciconfigstatus) | Status of linting, can be either valid or invalid. | @@ -9281,6 +9394,20 @@ Represents the total number of issues and their weights for a particular day. | <a id="ciconfiggroupname"></a>`name` | [`String`](#string) | Name of the job group. | | <a id="ciconfiggroupsize"></a>`size` | [`Int`](#int) | Size of the job group. | +### `CiConfigInclude` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciconfigincludeblob"></a>`blob` | [`String`](#string) | File blob location. It can be masked if it contains masked variables, e.g., "https://gitlab.com/gitlab-org/gitlab/-/blob/e52d6d0246d7375291850e61f0abc101fbda9dc2/.gitlab/ci/build-images.gitlab-ci.yml". | +| <a id="ciconfigincludecontextproject"></a>`contextProject` | [`String`](#string) | Current project scope, e.g., "gitlab-org/gitlab". | +| <a id="ciconfigincludecontextsha"></a>`contextSha` | [`String`](#string) | Current sha scope. | +| <a id="ciconfigincludeextra"></a>`extra` | [`JSON`](#json) | Extra information for the `include`, which can contain `job_name`, `project`, and `ref`. Values can be masked if they contain masked variables. | +| <a id="ciconfigincludelocation"></a>`location` | [`String`](#string) | File location. It can be masked if it contains masked variables, e.g., ".gitlab/ci/build-images.gitlab-ci.yml". | +| <a id="ciconfigincluderaw"></a>`raw` | [`String`](#string) | File raw location. It can be masked if it contains masked variables, e.g., "https://gitlab.com/gitlab-org/gitlab/-/raw/e52d6d0246d7375291850e61f0abc101fbda9dc2/.gitlab/ci/build-images.gitlab-ci.yml". | +| <a id="ciconfigincludetype"></a>`type` | [`CiConfigIncludeType`](#ciconfigincludetype) | Include type. | + ### `CiConfigJob` #### Fields @@ -9407,8 +9534,8 @@ Represents the total number of issues and their weights for a particular day. | <a id="ciminutesnamespacemonthlyusageminutes"></a>`minutes` | [`Int`](#int) | Total number of minutes used by all projects in the namespace. | | <a id="ciminutesnamespacemonthlyusagemonth"></a>`month` | [`String`](#string) | Month related to the usage data. | | <a id="ciminutesnamespacemonthlyusagemonthiso8601"></a>`monthIso8601` | [`ISO8601Date`](#iso8601date) | Month related to the usage data in ISO 8601 date format. | -| <a id="ciminutesnamespacemonthlyusageprojects"></a>`projects` | [`CiMinutesProjectMonthlyUsageConnection`](#ciminutesprojectmonthlyusageconnection) | CI minutes usage data for projects in the namespace. (see [Connections](#connections)) | -| <a id="ciminutesnamespacemonthlyusagesharedrunnersduration"></a>`sharedRunnersDuration` | [`Int`](#int) | Total numbers of minutes used by the shared runners in the namespace. | +| <a id="ciminutesnamespacemonthlyusageprojects"></a>`projects` | [`CiMinutesProjectMonthlyUsageConnection`](#ciminutesprojectmonthlyusageconnection) | CI/CD minutes usage data for projects in the namespace. (see [Connections](#connections)) | +| <a id="ciminutesnamespacemonthlyusagesharedrunnersduration"></a>`sharedRunnersDuration` | [`Int`](#int) | Total duration (in seconds) of shared runners use by the namespace for the month. | ### `CiMinutesProjectMonthlyUsage` @@ -9416,8 +9543,9 @@ Represents the total number of issues and their weights for a particular day. | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="ciminutesprojectmonthlyusageminutes"></a>`minutes` | [`Int`](#int) | Number of CI minutes used by the project in the month. | +| <a id="ciminutesprojectmonthlyusageminutes"></a>`minutes` | [`Int`](#int) | Number of CI/CD minutes used by the project in the month. | | <a id="ciminutesprojectmonthlyusagename"></a>`name` | [`String`](#string) | Name of the project. | +| <a id="ciminutesprojectmonthlyusagesharedrunnersduration"></a>`sharedRunnersDuration` | [`Int`](#int) | Total duration (in seconds) of shared runners use by the project for the month. | ### `CiRunner` @@ -9428,18 +9556,21 @@ Represents the total number of issues and their weights for a particular day. | <a id="cirunneraccesslevel"></a>`accessLevel` | [`CiRunnerAccessLevel!`](#cirunneraccesslevel) | Access level of the runner. | | <a id="cirunneractive"></a>`active` **{warning-solid}** | [`Boolean!`](#boolean) | **Deprecated** in 14.8. Use paused. | | <a id="cirunneradminurl"></a>`adminUrl` | [`String`](#string) | Admin URL of the runner. Only available for administrators. | +| <a id="cirunnerarchitecturename"></a>`architectureName` | [`String`](#string) | Architecture provided by the the runner. | | <a id="cirunnercontactedat"></a>`contactedAt` | [`Time`](#time) | Timestamp of last contact from this runner. | | <a id="cirunnercreatedat"></a>`createdAt` | [`Time`](#time) | Timestamp of creation of this runner. | | <a id="cirunnerdescription"></a>`description` | [`String`](#string) | Description of the runner. | | <a id="cirunnereditadminurl"></a>`editAdminUrl` | [`String`](#string) | Admin form URL of the runner. Only available for administrators. | -| <a id="cirunnerexecutorname"></a>`executorName` | [`String`](#string) | Executor last advertised by the runner. Available only when feature flag `graphql_ci_runner_executor` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | +| <a id="cirunnerexecutorname"></a>`executorName` | [`String`](#string) | Executor last advertised by the runner. | | <a id="cirunnergroups"></a>`groups` | [`GroupConnection`](#groupconnection) | Groups the runner is associated with. For group runners only. (see [Connections](#connections)) | | <a id="cirunnerid"></a>`id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner. | | <a id="cirunneripaddress"></a>`ipAddress` | [`String`](#string) | IP address of the runner. | | <a id="cirunnerjobcount"></a>`jobCount` | [`Int`](#int) | Number of jobs processed by the runner (limited to 1000, plus one to indicate that more items exist). | | <a id="cirunnerlocked"></a>`locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | +| <a id="cirunnermaintenancenote"></a>`maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | | <a id="cirunnermaximumtimeout"></a>`maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | | <a id="cirunnerpaused"></a>`paused` | [`Boolean!`](#boolean) | Indicates the runner is paused and not available to run jobs. | +| <a id="cirunnerplatformname"></a>`platformName` | [`String`](#string) | Platform provided by the runner. | | <a id="cirunnerprivateprojectsminutescostfactor"></a>`privateProjectsMinutesCostFactor` | [`Float`](#float) | Private projects' "minutes cost factor" associated with the runner (GitLab.com only). | | <a id="cirunnerprojectcount"></a>`projectCount` | [`Int`](#int) | Number of projects that the runner is associated with. | | <a id="cirunnerprojects"></a>`projects` | [`ProjectConnection`](#projectconnection) | Projects the runner is associated with. For project runners only. (see [Connections](#connections)) | @@ -9482,7 +9613,7 @@ Returns [`CiRunnerStatus!`](#cirunnerstatus). | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="cirunnerstatuslegacymode"></a>`legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 14.6. Will be removed in 15.0. From that release onward, the field will behave as if legacyMode is null. | +| <a id="cirunnerstatuslegacymode"></a>`legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. In GitLab 16.0 and later, the field will act as if `legacyMode` is null. | ### `CiStage` @@ -9772,6 +9903,7 @@ A container repository. | <a id="containerrepositoryexpirationpolicycleanupstatus"></a>`expirationPolicyCleanupStatus` | [`ContainerRepositoryCleanupStatus`](#containerrepositorycleanupstatus) | Tags cleanup status for the container repository. | | <a id="containerrepositoryexpirationpolicystartedat"></a>`expirationPolicyStartedAt` | [`Time`](#time) | Timestamp when the cleanup done by the expiration policy was started on the container repository. | | <a id="containerrepositoryid"></a>`id` | [`ID!`](#id) | ID of the container repository. | +| <a id="containerrepositorylastcleanupdeletedtagscount"></a>`lastCleanupDeletedTagsCount` | [`Int`](#int) | Number of deleted tags from the last cleanup. | | <a id="containerrepositorylocation"></a>`location` | [`String!`](#string) | URL of the container repository. | | <a id="containerrepositorymigrationstate"></a>`migrationState` | [`String!`](#string) | Migration state of the container repository. | | <a id="containerrepositoryname"></a>`name` | [`String!`](#string) | Name of the container repository. | @@ -9794,6 +9926,7 @@ Details of a container repository. | <a id="containerrepositorydetailsexpirationpolicycleanupstatus"></a>`expirationPolicyCleanupStatus` | [`ContainerRepositoryCleanupStatus`](#containerrepositorycleanupstatus) | Tags cleanup status for the container repository. | | <a id="containerrepositorydetailsexpirationpolicystartedat"></a>`expirationPolicyStartedAt` | [`Time`](#time) | Timestamp when the cleanup done by the expiration policy was started on the container repository. | | <a id="containerrepositorydetailsid"></a>`id` | [`ID!`](#id) | ID of the container repository. | +| <a id="containerrepositorydetailslastcleanupdeletedtagscount"></a>`lastCleanupDeletedTagsCount` | [`Int`](#int) | Number of deleted tags from the last cleanup. | | <a id="containerrepositorydetailslocation"></a>`location` | [`String!`](#string) | URL of the container repository. | | <a id="containerrepositorydetailsmigrationstate"></a>`migrationState` | [`String!`](#string) | Migration state of the container repository. | | <a id="containerrepositorydetailsname"></a>`name` | [`String!`](#string) | Name of the container repository. | @@ -9896,6 +10029,7 @@ A custom emoji uploaded by user. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="customerrelationscontactactive"></a>`active` | [`Boolean!`](#boolean) | State of the contact. | | <a id="customerrelationscontactcreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp the contact was created. | | <a id="customerrelationscontactdescription"></a>`description` | [`String`](#string) | Description of or notes for the contact. | | <a id="customerrelationscontactemail"></a>`email` | [`String`](#string) | Email address of the contact. | @@ -9912,6 +10046,7 @@ A custom emoji uploaded by user. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="customerrelationsorganizationactive"></a>`active` | [`Boolean!`](#boolean) | State of the organization. | | <a id="customerrelationsorganizationcreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp the organization was created. | | <a id="customerrelationsorganizationdefaultrate"></a>`defaultRate` | [`Float`](#float) | Standard billing rate for the organization. | | <a id="customerrelationsorganizationdescription"></a>`description` | [`String`](#string) | Description of or notes for the organization. | @@ -10529,7 +10664,7 @@ Returns [`[DoraMetric!]`](#dorametric). | Name | Type | Description | | ---- | ---- | ----------- | | <a id="dorametricdate"></a>`date` | [`String`](#string) | Date of the data point. | -| <a id="dorametricvalue"></a>`value` | [`Int`](#int) | Value of the data point. | +| <a id="dorametricvalue"></a>`value` | [`Float`](#float) | Value of the data point. | ### `Environment` @@ -11231,6 +11366,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="groupactualrepositorysizelimit"></a>`actualRepositorySizeLimit` | [`Float`](#float) | Size limit for repositories in the namespace in bytes. | | <a id="groupadditionalpurchasedstoragesize"></a>`additionalPurchasedStorageSize` | [`Float`](#float) | Additional storage purchased for the root namespace in bytes. | +| <a id="groupallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean!`](#boolean) | Indicates whether to regularly prune stale group runners. Defaults to false. | | <a id="groupautodevopsenabled"></a>`autoDevopsEnabled` | [`Boolean`](#boolean) | Indicates whether Auto DevOps is enabled for all projects within this group. | | <a id="groupavatarurl"></a>`avatarUrl` | [`String`](#string) | Avatar URL of the group. | | <a id="groupcontacts"></a>`contacts` | [`CustomerRelationsContactConnection`](#customerrelationscontactconnection) | Find contacts of this group. (see [Connections](#connections)) | @@ -11489,6 +11625,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="groupgroupmembersaccesslevels"></a>`accessLevels` | [`[AccessLevelEnum!]`](#accesslevelenum) | Filter members by the given access levels. | | <a id="groupgroupmembersrelations"></a>`relations` | [`[GroupMemberRelation!]`](#groupmemberrelation) | Filter members by the given member relations. | | <a id="groupgroupmemberssearch"></a>`search` | [`String`](#string) | Search query. | @@ -11515,6 +11652,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupissuesconfidential"></a>`confidential` | [`Boolean`](#boolean) | Filter for confidential issues. If "false", excludes confidential issues. If "true", returns only confidential issues. | | <a id="groupissuescreatedafter"></a>`createdAfter` | [`Time`](#time) | Issues created after this date. | | <a id="groupissuescreatedbefore"></a>`createdBefore` | [`Time`](#time) | Issues created before this date. | +| <a id="groupissuescrmcontactid"></a>`crmContactId` | [`String`](#string) | ID of a contact assigned to the issues. | +| <a id="groupissuescrmorganizationid"></a>`crmOrganizationId` | [`String`](#string) | ID of an organization assigned to the issues. | | <a id="groupissuesepicid"></a>`epicId` | [`String`](#string) | ID of an epic associated with the issues, "none" and "any" values are supported. | | <a id="groupissuesiid"></a>`iid` | [`String`](#string) | IID of the issue. For example, "1". | | <a id="groupissuesiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues. For example, `["1", "2"]`. | @@ -11551,7 +11690,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="groupiterationcadencesactive"></a>`active` | [`Boolean`](#boolean) | Whether the iteration cadence is active. | -| <a id="groupiterationcadencesautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate future iterations. | +| <a id="groupiterationcadencesautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate upcoming iterations. | | <a id="groupiterationcadencesdurationinweeks"></a>`durationInWeeks` | [`Int`](#int) | Duration in weeks of the iterations within this cadence. | | <a id="groupiterationcadencesid"></a>`id` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iteration cadence to look up. | | <a id="groupiterationcadencesincludeancestorgroups"></a>`includeAncestorGroups` | [`Boolean`](#boolean) | Whether to include ancestor groups to search iterations cadences in. | @@ -11756,6 +11895,23 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="grouprunnerstaglist"></a>`tagList` | [`[String!]`](#string) | Filter by tags associated with the runner (comma-separated or array). | | <a id="grouprunnerstype"></a>`type` | [`CiRunnerType`](#cirunnertype) | Filter runners by type. | +##### `Group.scanExecutionPolicies` + +Scan Execution Policies of the namespace. + +Returns [`ScanExecutionPolicyConnection`](#scanexecutionpolicyconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="groupscanexecutionpoliciesactionscantypes"></a>`actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`. | +| <a id="groupscanexecutionpoliciesrelationship"></a>`relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | + ##### `Group.timelogs` Time logged on issues and merge requests in the group and its subgroups. @@ -11922,6 +12078,17 @@ Contains release-related statistics about a group. | <a id="groupreleasestatsreleasescount"></a>`releasesCount` | [`Int`](#int) | Total number of releases in all descendant projects of the group. | | <a id="groupreleasestatsreleasespercentage"></a>`releasesPercentage` | [`Int`](#int) | Percentage of the group's descendant projects that have at least one release. | +### `GroupSecurityPolicySource` + +Represents the source of a security policy belonging to a group. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="groupsecuritypolicysourceinherited"></a>`inherited` | [`Boolean!`](#boolean) | Indicates whether this policy is inherited from parent group. | +| <a id="groupsecuritypolicysourcenamespace"></a>`namespace` | [`Namespace`](#namespace) | Project the policy is associated with. | + ### `GroupStats` Contains statistics about a group. @@ -12260,11 +12427,11 @@ Represents an iteration cadence. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="iterationcadenceactive"></a>`active` | [`Boolean`](#boolean) | Whether the iteration cadence is active. | -| <a id="iterationcadenceautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate future iterations. | +| <a id="iterationcadenceautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate upcoming iterations. | | <a id="iterationcadencedescription"></a>`description` | [`String`](#string) | Description of the iteration cadence. Maximum length is 5000 characters. | | <a id="iterationcadencedurationinweeks"></a>`durationInWeeks` | [`Int`](#int) | Duration in weeks of the iterations within this cadence. | | <a id="iterationcadenceid"></a>`id` | [`IterationsCadenceID!`](#iterationscadenceid) | Global ID of the iteration cadence. | -| <a id="iterationcadenceiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Future iterations to be created when iteration cadence is set to automatic. | +| <a id="iterationcadenceiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="iterationcadencerollover"></a>`rollOver` | [`Boolean!`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | | <a id="iterationcadencestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | | <a id="iterationcadencetitle"></a>`title` | [`String!`](#string) | Title of the iteration cadence. | @@ -12464,7 +12631,6 @@ Maven metadata. | <a id="mergerequestconflicts"></a>`conflicts` | [`Boolean!`](#boolean) | Indicates if the merge request has conflicts. | | <a id="mergerequestcreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp of when the merge request was created. | | <a id="mergerequestdefaultmergecommitmessage"></a>`defaultMergeCommitMessage` | [`String`](#string) | Default merge commit message of the merge request. | -| <a id="mergerequestdefaultmergecommitmessagewithdescription"></a>`defaultMergeCommitMessageWithDescription` **{warning-solid}** | [`String`](#string) | **Deprecated** in 14.5. Define merge commit template in project and use `defaultMergeCommitMessage`. | | <a id="mergerequestdefaultsquashcommitmessage"></a>`defaultSquashCommitMessage` | [`String`](#string) | Default squash commit message of the merge request. | | <a id="mergerequestdescription"></a>`description` | [`String`](#string) | Description of the merge request (Markdown rendered as HTML for caching). | | <a id="mergerequestdescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | @@ -13748,6 +13914,32 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="namespaceprojectssearch"></a>`search` | [`String`](#string) | Search project with most similar names or paths. | | <a id="namespaceprojectssort"></a>`sort` | [`NamespaceProjectSort`](#namespaceprojectsort) | Sort projects by this criteria. | +##### `Namespace.scanExecutionPolicies` + +Scan Execution Policies of the namespace. + +Returns [`ScanExecutionPolicyConnection`](#scanexecutionpolicyconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="namespacescanexecutionpoliciesactionscantypes"></a>`actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`. | +| <a id="namespacescanexecutionpoliciesrelationship"></a>`relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | + +### `NamespaceCiCdSetting` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="namespacecicdsettingallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean`](#boolean) | Indicates if stale runners directly belonging to this namespace should be periodically pruned. | +| <a id="namespacecicdsettingnamespace"></a>`namespace` | [`Namespace`](#namespace) | Namespace the CI/CD settings belong to. | + ### `NetworkPolicy` Represents the network policy. @@ -13853,7 +14045,7 @@ Active period time range for on-call rotation. ### `Package` -Represents a package in the Package Registry. Note that this type is in beta and susceptible to changes. +Represents a package with pipelines in the Package Registry. #### Fields @@ -13865,13 +14057,32 @@ Represents a package in the Package Registry. Note that this type is in beta and | <a id="packagemetadata"></a>`metadata` | [`PackageMetadata`](#packagemetadata) | Package metadata. | | <a id="packagename"></a>`name` | [`String!`](#string) | Name of the package. | | <a id="packagepackagetype"></a>`packageType` | [`PackageTypeEnum!`](#packagetypeenum) | Package type. | -| <a id="packagepipelines"></a>`pipelines` **{warning-solid}** | [`PipelineConnection`](#pipelineconnection) | **Deprecated** in 14.6. Due to scalability concerns, this field is going to be removed. | +| <a id="packagepipelines"></a>`pipelines` | [`PipelineConnection`](#pipelineconnection) | Pipelines that built the package. Max page size 20. (see [Connections](#connections)) | | <a id="packageproject"></a>`project` | [`Project!`](#project) | Project where the package is stored. | | <a id="packagestatus"></a>`status` | [`PackageStatus!`](#packagestatus) | Package status. | | <a id="packagetags"></a>`tags` | [`PackageTagConnection`](#packagetagconnection) | Package tags. (see [Connections](#connections)) | | <a id="packageupdatedat"></a>`updatedAt` | [`Time!`](#time) | Date of most recent update. | | <a id="packageversion"></a>`version` | [`String`](#string) | Version string. | -| <a id="packageversions"></a>`versions` **{warning-solid}** | [`PackageConnection`](#packageconnection) | **Deprecated** in 13.11. This field is now only returned in the PackageDetailsType. | + +### `PackageBase` + +Represents a package in the Package Registry. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="packagebasecandestroy"></a>`canDestroy` | [`Boolean!`](#boolean) | Whether the user can destroy the package. | +| <a id="packagebasecreatedat"></a>`createdAt` | [`Time!`](#time) | Date of creation. | +| <a id="packagebaseid"></a>`id` | [`PackagesPackageID!`](#packagespackageid) | ID of the package. | +| <a id="packagebasemetadata"></a>`metadata` | [`PackageMetadata`](#packagemetadata) | Package metadata. | +| <a id="packagebasename"></a>`name` | [`String!`](#string) | Name of the package. | +| <a id="packagebasepackagetype"></a>`packageType` | [`PackageTypeEnum!`](#packagetypeenum) | Package type. | +| <a id="packagebaseproject"></a>`project` | [`Project!`](#project) | Project where the package is stored. | +| <a id="packagebasestatus"></a>`status` | [`PackageStatus!`](#packagestatus) | Package status. | +| <a id="packagebasetags"></a>`tags` | [`PackageTagConnection`](#packagetagconnection) | Package tags. (see [Connections](#connections)) | +| <a id="packagebaseupdatedat"></a>`updatedAt` | [`Time!`](#time) | Date of most recent update. | +| <a id="packagebaseversion"></a>`version` | [`String`](#string) | Version string. | ### `PackageComposerJsonType` @@ -13913,7 +14124,7 @@ Represents a package dependency link. ### `PackageDetailsType` -Represents a package details in the Package Registry. Note that this type is in beta and susceptible to changes. +Represents a package details in the Package Registry. #### Fields @@ -13933,7 +14144,7 @@ Represents a package details in the Package Registry. Note that this type is in | <a id="packagedetailstypenugeturl"></a>`nugetUrl` | [`String`](#string) | Url of the Nuget project endpoint. | | <a id="packagedetailstypepackagefiles"></a>`packageFiles` | [`PackageFileConnection`](#packagefileconnection) | Package files. (see [Connections](#connections)) | | <a id="packagedetailstypepackagetype"></a>`packageType` | [`PackageTypeEnum!`](#packagetypeenum) | Package type. | -| <a id="packagedetailstypepipelines"></a>`pipelines` **{warning-solid}** | [`PipelineConnection`](#pipelineconnection) | **Deprecated** in 14.6. Due to scalability concerns, this field is going to be removed. | +| <a id="packagedetailstypepipelines"></a>`pipelines` | [`PipelineConnection`](#pipelineconnection) | Pipelines that built the package. Max page size 20. (see [Connections](#connections)) | | <a id="packagedetailstypeproject"></a>`project` | [`Project!`](#project) | Project where the package is stored. | | <a id="packagedetailstypepypisetupurl"></a>`pypiSetupUrl` | [`String`](#string) | Url of the PyPi project setup endpoint. | | <a id="packagedetailstypepypiurl"></a>`pypiUrl` | [`String`](#string) | Url of the PyPi project endpoint. | @@ -13941,7 +14152,7 @@ Represents a package details in the Package Registry. Note that this type is in | <a id="packagedetailstypetags"></a>`tags` | [`PackageTagConnection`](#packagetagconnection) | Package tags. (see [Connections](#connections)) | | <a id="packagedetailstypeupdatedat"></a>`updatedAt` | [`Time!`](#time) | Date of most recent update. | | <a id="packagedetailstypeversion"></a>`version` | [`String`](#string) | Version string. | -| <a id="packagedetailstypeversions"></a>`versions` | [`PackageConnection`](#packageconnection) | Other versions of the package. (see [Connections](#connections)) | +| <a id="packagedetailstypeversions"></a>`versions` | [`PackageBaseConnection`](#packagebaseconnection) | Other versions of the package. (see [Connections](#connections)) | ### `PackageFile` @@ -14752,6 +14963,8 @@ Returns [`Issue`](#issue). | <a id="projectissueconfidential"></a>`confidential` | [`Boolean`](#boolean) | Filter for confidential issues. If "false", excludes confidential issues. If "true", returns only confidential issues. | | <a id="projectissuecreatedafter"></a>`createdAfter` | [`Time`](#time) | Issues created after this date. | | <a id="projectissuecreatedbefore"></a>`createdBefore` | [`Time`](#time) | Issues created before this date. | +| <a id="projectissuecrmcontactid"></a>`crmContactId` | [`String`](#string) | ID of a contact assigned to the issues. | +| <a id="projectissuecrmorganizationid"></a>`crmOrganizationId` | [`String`](#string) | ID of an organization assigned to the issues. | | <a id="projectissueepicid"></a>`epicId` | [`String`](#string) | ID of an epic associated with the issues, "none" and "any" values are supported. | | <a id="projectissueiid"></a>`iid` | [`String`](#string) | IID of the issue. For example, "1". | | <a id="projectissueiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues. For example, `["1", "2"]`. | @@ -14792,6 +15005,8 @@ Returns [`IssueStatusCountsType`](#issuestatuscountstype). | <a id="projectissuestatuscountsconfidential"></a>`confidential` | [`Boolean`](#boolean) | Filter for confidential issues. If "false", excludes confidential issues. If "true", returns only confidential issues. | | <a id="projectissuestatuscountscreatedafter"></a>`createdAfter` | [`Time`](#time) | Issues created after this date. | | <a id="projectissuestatuscountscreatedbefore"></a>`createdBefore` | [`Time`](#time) | Issues created before this date. | +| <a id="projectissuestatuscountscrmcontactid"></a>`crmContactId` | [`String`](#string) | ID of a contact assigned to the issues. | +| <a id="projectissuestatuscountscrmorganizationid"></a>`crmOrganizationId` | [`String`](#string) | ID of an organization assigned to the issues. | | <a id="projectissuestatuscountsiid"></a>`iid` | [`String`](#string) | IID of the issue. For example, "1". | | <a id="projectissuestatuscountsiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues. For example, `["1", "2"]`. | | <a id="projectissuestatuscountslabelname"></a>`labelName` | [`[String]`](#string) | Labels applied to this issue. | @@ -14829,6 +15044,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="projectissuesconfidential"></a>`confidential` | [`Boolean`](#boolean) | Filter for confidential issues. If "false", excludes confidential issues. If "true", returns only confidential issues. | | <a id="projectissuescreatedafter"></a>`createdAfter` | [`Time`](#time) | Issues created after this date. | | <a id="projectissuescreatedbefore"></a>`createdBefore` | [`Time`](#time) | Issues created before this date. | +| <a id="projectissuescrmcontactid"></a>`crmContactId` | [`String`](#string) | ID of a contact assigned to the issues. | +| <a id="projectissuescrmorganizationid"></a>`crmOrganizationId` | [`String`](#string) | ID of an organization assigned to the issues. | | <a id="projectissuesepicid"></a>`epicId` | [`String`](#string) | ID of an epic associated with the issues, "none" and "any" values are supported. | | <a id="projectissuesiid"></a>`iid` | [`String`](#string) | IID of the issue. For example, "1". | | <a id="projectissuesiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of issues. For example, `["1", "2"]`. | @@ -14865,7 +15082,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectiterationcadencesactive"></a>`active` | [`Boolean`](#boolean) | Whether the iteration cadence is active. | -| <a id="projectiterationcadencesautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate future iterations. | +| <a id="projectiterationcadencesautomatic"></a>`automatic` | [`Boolean`](#boolean) | Whether the iteration cadence should automatically generate upcoming iterations. | | <a id="projectiterationcadencesdurationinweeks"></a>`durationInWeeks` | [`Int`](#int) | Duration in weeks of the iterations within this cadence. | | <a id="projectiterationcadencesid"></a>`id` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iteration cadence to look up. | | <a id="projectiterationcadencesincludeancestorgroups"></a>`includeAncestorGroups` | [`Boolean`](#boolean) | Whether to include ancestor groups to search iterations cadences in. | @@ -15019,7 +15236,7 @@ Network Policies of the project. WARNING: **Deprecated** in 14.8. -Network policies are deprecated and will be removed in GitLab 15.0. +Network policies are deprecated and will be removed in GitLab 16.0. Returns [`NetworkPolicyConnection`](#networkpolicyconnection). @@ -15203,6 +15420,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectscanexecutionpoliciesactionscantypes"></a>`actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`. | +| <a id="projectscanexecutionpoliciesrelationship"></a>`relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | ##### `Project.securityTrainingProviders` @@ -15227,6 +15445,7 @@ Returns [`[SecurityTrainingUrl!]`](#securitytrainingurl). | Name | Type | Description | | ---- | ---- | ----------- | | <a id="projectsecuritytrainingurlsidentifierexternalids"></a>`identifierExternalIds` | [`[String!]!`](#string) | List of external IDs of vulnerability identifiers. | +| <a id="projectsecuritytrainingurlslanguage"></a>`language` | [`String`](#string) | Desired language for training urls. | ##### `Project.sentryDetailedError` @@ -15481,6 +15700,16 @@ Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction). | <a id="projectpermissionsupdatewiki"></a>`updateWiki` | [`Boolean!`](#boolean) | Indicates the user can perform `update_wiki` on this resource. | | <a id="projectpermissionsuploadfile"></a>`uploadFile` | [`Boolean!`](#boolean) | Indicates the user can perform `upload_file` on this resource. | +### `ProjectSecurityPolicySource` + +Represents the source of a security policy belonging to a project. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectsecuritypolicysourceproject"></a>`project` | [`Project`](#project) | Project the policy is associated with. | + ### `ProjectSecurityTraining` #### Fields @@ -15503,6 +15732,7 @@ Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction). | ---- | ---- | ----------- | | <a id="projectstatisticsbuildartifactssize"></a>`buildArtifactsSize` | [`Float!`](#float) | Build artifacts size of the project in bytes. | | <a id="projectstatisticscommitcount"></a>`commitCount` | [`Float!`](#float) | Commit count of the project. | +| <a id="projectstatisticscontainerregistrysize"></a>`containerRegistrySize` | [`Float`](#float) | Container Registry size of the project in bytes. | | <a id="projectstatisticslfsobjectssize"></a>`lfsObjectsSize` | [`Float!`](#float) | Large File Storage (LFS) object size of the project in bytes. | | <a id="projectstatisticspackagessize"></a>`packagesSize` | [`Float!`](#float) | Packages size of the project in bytes. | | <a id="projectstatisticspipelineartifactssize"></a>`pipelineArtifactsSize` | [`Float`](#float) | CI Pipeline artifacts size in bytes. | @@ -15849,6 +16079,7 @@ Counts of requirements by their state. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="rootstoragestatisticsbuildartifactssize"></a>`buildArtifactsSize` | [`Float!`](#float) | CI artifacts size in bytes. | +| <a id="rootstoragestatisticscontainerregistrysize"></a>`containerRegistrySize` | [`Float!`](#float) | Container Registry size in bytes. | | <a id="rootstoragestatisticsdependencyproxysize"></a>`dependencyProxySize` | [`Float!`](#float) | Dependency Proxy sizes in bytes. | | <a id="rootstoragestatisticslfsobjectssize"></a>`lfsObjectsSize` | [`Float!`](#float) | LFS objects size in bytes. | | <a id="rootstoragestatisticspackagessize"></a>`packagesSize` | [`Float!`](#float) | Packages size in bytes. | @@ -15985,6 +16216,7 @@ Represents the scan execution policy. | <a id="scanexecutionpolicydescription"></a>`description` | [`String!`](#string) | Description of the policy. | | <a id="scanexecutionpolicyenabled"></a>`enabled` | [`Boolean!`](#boolean) | Indicates whether this policy is enabled. | | <a id="scanexecutionpolicyname"></a>`name` | [`String!`](#string) | Name of the policy. | +| <a id="scanexecutionpolicysource"></a>`source` | [`SecurityPolicySource!`](#securitypolicysource) | Source of the policy. Its fields depend on the source type. | | <a id="scanexecutionpolicyupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp of when the policy YAML was last updated. | | <a id="scanexecutionpolicyyaml"></a>`yaml` | [`String!`](#string) | YAML definition of the policy. | @@ -16612,6 +16844,7 @@ Describes an incident management timeline event. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="timelogid"></a>`id` | [`ID!`](#id) | Internal ID of the timelog. | | <a id="timelogissue"></a>`issue` | [`Issue`](#issue) | Issue that logged time was added to. | | <a id="timelogmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request that logged time was added to. | | <a id="timelognote"></a>`note` | [`Note`](#note) | Note where the quick action was executed to add the logged time. | @@ -16619,6 +16852,15 @@ Describes an incident management timeline event. | <a id="timelogsummary"></a>`summary` | [`String`](#string) | Summary of how the time was spent. | | <a id="timelogtimespent"></a>`timeSpent` | [`Int!`](#int) | Time spent displayed in seconds. | | <a id="timeloguser"></a>`user` | [`UserCore!`](#usercore) | User that logged the time. | +| <a id="timeloguserpermissions"></a>`userPermissions` | [`TimelogPermissions!`](#timelogpermissions) | Permissions for the current user on the resource. | + +### `TimelogPermissions` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="timelogpermissionsadmintimelog"></a>`adminTimelog` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_timelog` on this resource. | ### `Todo` @@ -16650,6 +16892,7 @@ Representing a to-do entry. | <a id="topicdescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | | <a id="topicid"></a>`id` | [`ID!`](#id) | ID of the topic. | | <a id="topicname"></a>`name` | [`String!`](#string) | Name of the topic. | +| <a id="topictitle"></a>`title` | [`String!`](#string) | Title of the topic. | ### `Tree` @@ -17582,8 +17825,21 @@ Represents vulnerability letter grades with associated projects. | <a id="workitemstate"></a>`state` | [`WorkItemState!`](#workitemstate) | State of the work item. | | <a id="workitemtitle"></a>`title` | [`String!`](#string) | Title of the work item. | | <a id="workitemtitlehtml"></a>`titleHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `title`. | +| <a id="workitemuserpermissions"></a>`userPermissions` | [`WorkItemPermissions!`](#workitempermissions) | Permissions for the current user on the resource. | | <a id="workitemworkitemtype"></a>`workItemType` | [`WorkItemType!`](#workitemtype) | Type assigned to the work item. | +### `WorkItemPermissions` + +Check permissions for the current user on a work item. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitempermissionsdeleteworkitem"></a>`deleteWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `delete_work_item` on this resource. | +| <a id="workitempermissionsreadworkitem"></a>`readWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `read_work_item` on this resource. | +| <a id="workitempermissionsupdateworkitem"></a>`updateWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `update_work_item` on this resource. | + ### `WorkItemType` #### Fields @@ -17662,7 +17918,7 @@ Filters the alerts based on given domain. | Value | Description | | ----- | ----------- | | <a id="alertmanagementdomainfilteroperations"></a>`operations` | Alerts for operations domain. | -| <a id="alertmanagementdomainfilterthreat_monitoring"></a>`threat_monitoring` | Alerts for threat monitoring domain. | +| <a id="alertmanagementdomainfilterthreat_monitoring"></a>`threat_monitoring` **{warning-solid}** | **Deprecated** in 15.0. Network policies are deprecated and will be removed in GitLab 16.0. | ### `AlertManagementIntegrationType` @@ -17773,6 +18029,17 @@ Types of blob viewers. | <a id="blobviewerstyperich"></a>`rich` | Rich blob viewers type. | | <a id="blobviewerstypesimple"></a>`simple` | Simple blob viewers type. | +### `CiConfigIncludeType` + +Include type. + +| Value | Description | +| ----- | ----------- | +| <a id="ciconfigincludetypefile"></a>`file` | Project file include. | +| <a id="ciconfigincludetypelocal"></a>`local` | Local include. | +| <a id="ciconfigincludetyperemote"></a>`remote` | Remote include. | +| <a id="ciconfigincludetypetemplate"></a>`template` | Template include. | + ### `CiConfigStatus` Values for YAML processor result. @@ -17830,12 +18097,11 @@ Values for sorting runners. | Value | Description | | ----- | ----------- | | <a id="cirunnerstatusactive"></a>`ACTIVE` **{warning-solid}** | **Deprecated** in 14.6. This was renamed. Use: [`CiRunner.paused`](#cirunnerpaused). | -| <a id="cirunnerstatusnever_contacted"></a>`NEVER_CONTACTED` | Runner that has never contacted this instance. Set legacyMode to null to utilize this value. Will replace NOT_CONNECTED starting in 15.0. | -| <a id="cirunnerstatusnot_connected"></a>`NOT_CONNECTED` **{warning-solid}** | **Deprecated** in 14.6. Use NEVER_CONTACTED instead. NEVER_CONTACTED will have a slightly different scope starting in 15.0, with STALE being returned instead after 3 months of no contact. | -| <a id="cirunnerstatusoffline"></a>`OFFLINE` **{warning-solid}** | **Deprecated** in 14.6. This field will have a slightly different scope starting in 15.0, with STALE being returned after a certain period offline. | +| <a id="cirunnerstatusnever_contacted"></a>`NEVER_CONTACTED` | Runner that has never contacted this instance. | +| <a id="cirunnerstatusoffline"></a>`OFFLINE` | Runner that has not contacted this instance within the last 2 hours. Will be considered `STALE` if offline for more than 3 months. | | <a id="cirunnerstatusonline"></a>`ONLINE` | Runner that contacted this instance within the last 2 hours. | | <a id="cirunnerstatuspaused"></a>`PAUSED` **{warning-solid}** | **Deprecated** in 14.6. This was renamed. Use: [`CiRunner.paused`](#cirunnerpaused). | -| <a id="cirunnerstatusstale"></a>`STALE` | Runner that has not contacted this instance within the last 3 months. Only available if legacyMode is null. Will be a possible return value starting in 15.0. | +| <a id="cirunnerstatusstale"></a>`STALE` | Runner that has not contacted this instance within the last 3 months. | ### `CiRunnerType` @@ -17849,9 +18115,11 @@ Values for sorting runners. | Value | Description | | ----- | ----------- | -| <a id="cirunnerupgradestatustypeavailable"></a>`AVAILABLE` | An update is available for the runner. | -| <a id="cirunnerupgradestatustypenot_available"></a>`NOT_AVAILABLE` | An update is not available for the runner. | -| <a id="cirunnerupgradestatustyperecommended"></a>`RECOMMENDED` | An update is available and recommended for the runner. | +| <a id="cirunnerupgradestatustypeavailable"></a>`AVAILABLE` | Upgrade is available for the runner. | +| <a id="cirunnerupgradestatustypeinvalid"></a>`INVALID` | Runner version is not valid. | +| <a id="cirunnerupgradestatustypenot_available"></a>`NOT_AVAILABLE` | Upgrade is not available for the runner. | +| <a id="cirunnerupgradestatustyperecommended"></a>`RECOMMENDED` | Upgrade is available and recommended for the runner. | +| <a id="cirunnerupgradestatustypeunknown"></a>`UNKNOWN` | Upgrade status is unknown. | ### `CodeQualityDegradationSeverity` @@ -18902,6 +19170,13 @@ The status of the security scan. | <a id="scanstatusreport_error"></a>`REPORT_ERROR` | The report artifact provided by the CI build couldn't be parsed. | | <a id="scanstatussucceeded"></a>`SUCCEEDED` | The report has been successfully prepared. | +### `SecurityPolicyRelationType` + +| Value | Description | +| ----- | ----------- | +| <a id="securitypolicyrelationtypedirect"></a>`DIRECT` | Policies defined for the project only. | +| <a id="securitypolicyrelationtypeinherited"></a>`INHERITED` | Policies defined for the project and project's ancestor groups. | + ### `SecurityReportTypeEnum` | Value | Description | @@ -19122,6 +19397,7 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumpersonal_access_token_expiry"></a>`PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for personal_access_token_expiry. | | <a id="usercalloutfeaturenameenumpipeline_needs_banner"></a>`PIPELINE_NEEDS_BANNER` | Callout feature name for pipeline_needs_banner. | | <a id="usercalloutfeaturenameenumpipeline_needs_hover_tip"></a>`PIPELINE_NEEDS_HOVER_TIP` | Callout feature name for pipeline_needs_hover_tip. | +| <a id="usercalloutfeaturenameenumpreview_user_over_limit_free_plan_alert"></a>`PREVIEW_USER_OVER_LIMIT_FREE_PLAN_ALERT` | Callout feature name for preview_user_over_limit_free_plan_alert. | | <a id="usercalloutfeaturenameenumprofile_personal_access_token_expiry"></a>`PROFILE_PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for profile_personal_access_token_expiry. | | <a id="usercalloutfeaturenameenumregistration_enabled_callout"></a>`REGISTRATION_ENABLED_CALLOUT` | Callout feature name for registration_enabled_callout. | | <a id="usercalloutfeaturenameenumsecurity_configuration_devops_alert"></a>`SECURITY_CONFIGURATION_DEVOPS_ALERT` | Callout feature name for security_configuration_devops_alert. | @@ -19142,6 +19418,7 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumtwo_factor_auth_recovery_settings_check"></a>`TWO_FACTOR_AUTH_RECOVERY_SETTINGS_CHECK` | Callout feature name for two_factor_auth_recovery_settings_check. | | <a id="usercalloutfeaturenameenumultimate_trial"></a>`ULTIMATE_TRIAL` | Callout feature name for ultimate_trial. | | <a id="usercalloutfeaturenameenumunfinished_tag_cleanup_callout"></a>`UNFINISHED_TAG_CLEANUP_CALLOUT` | Callout feature name for unfinished_tag_cleanup_callout. | +| <a id="usercalloutfeaturenameenumuser_reached_limit_free_plan_alert"></a>`USER_REACHED_LIMIT_FREE_PLAN_ALERT` | Callout feature name for user_reached_limit_free_plan_alert. | | <a id="usercalloutfeaturenameenumverification_reminder"></a>`VERIFICATION_REMINDER` | Callout feature name for verification_reminder. | | <a id="usercalloutfeaturenameenumweb_ide_alert_dismissed"></a>`WEB_IDE_ALERT_DISMISSED` | Callout feature name for web_ide_alert_dismissed. | | <a id="usercalloutfeaturenameenumweb_ide_ci_environments_guidance"></a>`WEB_IDE_CI_ENVIRONMENTS_GUIDANCE` | Callout feature name for web_ide_ci_environments_guidance. | @@ -19278,8 +19555,6 @@ Vulnerability sort values. | <a id="vulnerabilitysortseverity_desc"></a>`severity_desc` | Severity in descending order. | | <a id="vulnerabilitysortstate_asc"></a>`state_asc` | State in ascending order. | | <a id="vulnerabilitysortstate_desc"></a>`state_desc` | State in descending order. | -| <a id="vulnerabilitysorttitle_asc"></a>`title_asc` **{warning-solid}** | **Deprecated** in 14.2. Deprecated due to performance issues. | -| <a id="vulnerabilitysorttitle_desc"></a>`title_desc` **{warning-solid}** | **Deprecated** in 14.2. Deprecated due to performance issues. | ### `VulnerabilityState` @@ -19422,6 +19697,12 @@ A `ClustersClusterID` is a global ID. It is encoded as a string. An example `ClustersClusterID` is: `"gid://gitlab/Clusters::Cluster/1"`. +### `Color` + +Color represented as a hex code or named color. + +For example: "#fefefe". + ### `ComplianceManagementFrameworkID` A `ComplianceManagementFrameworkID` is a global ID. It is encoded as a string. @@ -19825,6 +20106,12 @@ For example: "2021-03-09T14:58:50+00:00". See `https://www.iso.org/iso-8601-date-and-time-format.html`. +### `TimelogID` + +A `TimelogID` is a global ID. It is encoded as a string. + +An example `TimelogID` is: `"gid://gitlab/Timelog/1"`. + ### `TodoID` A `TodoID` is a global ID. It is encoded as a string. @@ -19959,6 +20246,15 @@ One of: - [`NugetMetadata`](#nugetmetadata) - [`PypiMetadata`](#pypimetadata) +#### `SecurityPolicySource` + +Represents a policy source. Its fields depend on the source type. + +One of: + +- [`GroupSecurityPolicySource`](#groupsecuritypolicysource) +- [`ProjectSecurityPolicySource`](#projectsecuritypolicysource) + #### `VulnerabilityDetail` Represents a vulnerability detail field. The fields with data will depend on the vulnerability detail type. @@ -20954,3 +21250,13 @@ A time-frame defined as a closed inclusive range of two dates. | <a id="workitemconverttaskinputlockversion"></a>`lockVersion` | [`Int!`](#int) | Current lock version of the work item containing the task in the description. | | <a id="workitemconverttaskinputtitle"></a>`title` | [`String!`](#string) | Full string of the task to be replaced. New title for the created work item. | | <a id="workitemconverttaskinputworkitemtypeid"></a>`workItemTypeId` | [`WorkItemsTypeID!`](#workitemstypeid) | Global ID of the work item type used to create the new work item. | + +### `WorkItemDeletedTaskInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemdeletedtaskinputid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the task referenced in the work item's description. | +| <a id="workitemdeletedtaskinputlinenumberend"></a>`lineNumberEnd` | [`Int!`](#int) | Last line in the Markdown source that defines the list item task. | +| <a id="workitemdeletedtaskinputlinenumberstart"></a>`lineNumberStart` | [`Int!`](#int) | First line in the Markdown source that defines the list item task. | diff --git a/doc/api/graphql/removed_items.md b/doc/api/graphql/removed_items.md index f70add0de45..6778a0c2aab 100644 --- a/doc/api/graphql/removed_items.md +++ b/doc/api/graphql/removed_items.md @@ -10,6 +10,32 @@ GraphQL is a versionless API, unlike the REST API. Occasionally, items have to be updated or removed from the GraphQL API. According to our [process for removing items](index.md#deprecation-and-removal-process), here are the items that have been removed. +## GitLab 15.0 + +Fields removed in GitLab 15.0. + +### GraphQL Mutations + +[Removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85382) in GitLab 15.0: + +| Argument name | Mutation | Deprecated in | Use instead | +| -------------------- | -------------------- | ------------- | -------------------------- | +| - | `clusterAgentTokenDelete`| 14.7 | `clusterAgentTokenRevoke` | + +### GraphQL Fields + +[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/342882) in GitLab 15.0: + +| Argument name | Field name | Deprecated in | Use instead | +| -------------------- | --------------------| ------------- | -------------------------- | +| - | `pipelines` | 14.5 | None | + +### GraphQL Types + +| Field name | GraphQL type | Deprecated in | Use instead | +| ------------------------------------------ | ------------------------ | ------------- | ---------------------------------------------------------------------------------- | +| `defaultMergeCommitMessageWithDescription` | `GraphQL::Types::String` | 14.5 | None. Define a [merge commit template](../../user/project/merge_requests/commit_templates.md) in your project and use `defaultMergeCommitMessage`. | + ## GitLab 14.0 Fields [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63293) in GitLab 14.0: diff --git a/doc/api/groups.md b/doc/api/groups.md index e04e5207c95..b565e714285 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -455,7 +455,6 @@ Example response: "public_jobs":true, "build_timeout":3600, "auto_cancel_pending_pipelines":"enabled", - "build_coverage_regex":null, "ci_config_path":null, "shared_with_groups":[ { @@ -837,6 +836,7 @@ The `default_branch_protection` attribute determines whether users with the Deve | `0` | No protection. Users with the Developer or Maintainer role can: <br>- Push new commits<br>- Force push changes<br>- Delete the branch | | `1` | Partial protection. Users with the Developer or Maintainer role can: <br>- Push new commits | | `2` | Full protection. Only users with the Maintainer role can: <br>- Push new commits | +| `3` | Protected against pushes. Users with the Maintainer role can: <br>- Push new commits<br>- Force push changes<br>- Accept merge requests<br>Users with the Developer role can:<br>- Accept merge requests| ## New Subgroup diff --git a/doc/api/index.md b/doc/api/index.md index f78a501fb11..78e5f980679 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -348,7 +348,7 @@ The following table shows the possible return codes for API requests. | Return values | Description | |--------------------------|-------------| -| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, and the resource(s) itself is returned as JSON. | +| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, and the resource itself is returned as JSON. | | `204 No Content` | The server has successfully fulfilled the request, and there is no additional content to send in the response payload body. | | `201 Created` | The `POST` request was successful, and the resource is returned as JSON. | | `304 Not Modified` | The resource hasn't been modified since the last request. | diff --git a/doc/api/integrations.md b/doc/api/integrations.md index c1564826944..eaa826b3686 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -92,7 +92,6 @@ Parameters: | --------- | ---- | -------- | ----------- | | `api_key` | string | true | User API token. User must have access to task. All comments are attributed to this user. | | `restrict_to_branch` | string | false | Comma-separated list of branches to be are automatically inspected. Leave blank to include all branches. | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Asana integration @@ -128,7 +127,6 @@ Parameters: | --------- | ---- | -------- | ----------- | | `token` | string | true | The authentication token | `subdomain` | string | false | The subdomain setting | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Assembla integration @@ -169,7 +167,6 @@ Parameters: | `build_key` | string | true | Bamboo build plan key like KEY | | `username` | string | true | A user with API access, if applicable | | `password` | string | true | Password of the user | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Atlassian Bamboo CI integration @@ -206,9 +203,6 @@ Parameters: | `new_issue_url` | string | true | New Issue URL | | `issues_url` | string | true | Issue URL | | `project_url` | string | true | Project URL | -| `description` | string | false | Description | -| `title` | string | false | Title | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Bugzilla integration @@ -246,6 +240,8 @@ Parameters: | `project_url` | string | true | Pipeline URL. For example, `https://buildkite.com/example/pipeline` | | `enable_ssl_verification` | boolean | false | DEPRECATED: This parameter has no effect since SSL verification is always enabled | | `push_events` | boolean | false | Enable notifications for push events | +| `merge_requests_events` | boolean | false | Enable notifications for merge request events | +| `tag_push_events` | boolean | false | Enable notifications for tag push events | ### Disable Buildkite integration @@ -283,7 +279,6 @@ Parameters: | `token` | string | true | Campfire API token. To find it, log into Campfire and select **My info**. | | `subdomain` | string | false | Campfire subdomain. Text between `https://` and `.campfirenow.com` when you're logged in. | | `room` | string | false | Campfire room. The last part of the URL when you're in a room. | -| `push_events` | boolean | false | Enable notifications for push events. | ### Disable Campfire integration @@ -452,9 +447,6 @@ Parameters: | `new_issue_url` | string | true | New Issue URL | | `issues_url` | string | true | Issue URL | | `project_url` | string | true | Project URL | -| `description` | string | false | Description | -| `title` | string | false | Title | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Custom Issue Tracker integration @@ -709,7 +701,6 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `token` | string | true | Flowdock Git source token | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Flowdock integration @@ -832,7 +823,6 @@ Parameters: | `server_host` | string | false | localhost | | `server_port` | integer | false | 6659 | | `colorize_messages` | boolean | false | Colorize messages | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Irker (IRC gateway) integration @@ -1085,7 +1075,6 @@ Parameters: | --------- | ---- | -------- | ----------- | | `token` | string | true | The Pivotal Tracker token | | `restrict_to_branch` | boolean | false | Comma-separated list of branches to automatically inspect. Leave blank to include all branches. | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Pivotal Tracker integration @@ -1160,7 +1149,6 @@ Parameters: | `priority` | string | true | The priority | | `device` | string | false | Leave blank for all active devices | | `sound` | string | false | The sound of the notification | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Pushover integration @@ -1197,8 +1185,6 @@ Parameters: | `new_issue_url` | string | true | New Issue URL | | `project_url` | string | true | Project URL | | `issues_url` | string | true | Issue URL | -| `description` | string | false | Description | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable Redmine integration @@ -1404,6 +1390,7 @@ Parameters: | `username` | string | true | A user with permissions to trigger a manual build | | `password` | string | true | The password of the user | | `push_events` | boolean | false | Enable notifications for push events | +| `merge_requests_events` | boolean | false | Enable notifications for merge request events | ### Disable JetBrains TeamCity CI integration @@ -1554,8 +1541,6 @@ Parameters: | --------- | ---- | -------- | ----------- | | `issues_url` | string | true | Issue URL | | `project_url` | string | true | Project URL | -| `description` | string | false | Description | -| `push_events` | boolean | false | Enable notifications for push events | ### Disable YouTrack integration diff --git a/doc/api/issues.md b/doc/api/issues.md index e82aa8da8ed..44b947f14dc 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -1166,7 +1166,7 @@ PUT /projects/:id/issues/:issue_iid | Attribute | Type | Required | Description | |----------------|---------|----------|------------------------------------------------------------------------------------------------------------| | `add_labels` | string | no | Comma-separated label names to add to an issue. | -| `assignee_ids` | integer array | no | The ID of the user(s) to assign the issue to. Set to `0` or provide an empty value to unassign all assignees. | +| `assignee_ids` | integer array | no | The ID of the users to assign the issue to. Set to `0` or provide an empty value to unassign all assignees. | | `confidential` | boolean | no | Updates an issue to be confidential | | `description` | string | no | The description of an issue. Limited to 1,048,576 characters. | | `discussion_locked` | boolean | no | Flag indicating if the issue's discussion is locked. If the discussion is locked only project members can add or edit comments. | diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index 517ffde0046..ee9f1678b18 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -171,6 +171,9 @@ Download a single artifact file for a specific job of the latest successful pipeline for the given reference name from inside the job's artifacts archive. The file is extracted from the archive and streamed to the client. +The artifact file provides more detail than what is available in the +[CSV export](../user/application_security/vulnerability_report/index.md#export-vulnerability-details). + In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/201784) and later, artifacts for [parent and child pipelines](../ci/pipelines/parent_child_pipelines.md) are searched in hierarchical order from parent to child. For example, if both parent and child pipelines have a diff --git a/doc/api/linked_epics.md b/doc/api/linked_epics.md index df302be0555..0d2176dfc61 100644 --- a/doc/api/linked_epics.md +++ b/doc/api/linked_epics.md @@ -6,10 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Linked epics API **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352493) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `related_epics_widget`. Enabled by default. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `related_epics_widget`. On GitLab.com, this feature is available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352493) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `related_epics_widget`. Enabled by default. +> - [Feature flag `related_epics_widget`](https://gitlab.com/gitlab-org/gitlab/-/issues/357089) removed in GitLab 15.0. If the Related Epics feature is not available in your GitLab plan, a `403` status code is returned. diff --git a/doc/api/lint.md b/doc/api/lint.md index a271b75c035..c0d0b69dc77 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -268,7 +268,7 @@ a JSON payload, you can use `jq`. For example, create a file named `example-gitl ```yaml .api_test: rules: - - if: '$CI_PIPELINE_SOURCE=="merge_request_event"' + - if: $CI_PIPELINE_SOURCE=="merge_request_event" changes: - src/api/* deploy: diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index f2626574bf0..31f1cb41eb3 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Managed Licenses API **(ULTIMATE)** WARNING: -"approval" and "blacklisted" approval statuses are deprecated and scheduled to be changed to "allowed" and "denied" in GitLab 15.0. +"approval" and "blacklisted" approval statuses are changed to "allowed" and "denied" in GitLab 15.0. ## List managed licenses @@ -32,12 +32,12 @@ Example response: { "id": 1, "name": "MIT", - "approval_status": "approved" + "approval_status": "allowed" }, { "id": 3, "name": "ISC", - "approval_status": "blacklisted" + "approval_status": "denied" } ] ``` @@ -65,7 +65,7 @@ Example response: { "id": 1, "name": "MIT", - "approval_status": "blacklisted" + "approval_status": "denied" } ``` @@ -81,7 +81,7 @@ POST /projects/:id/managed_licenses | ------------- | ------- | -------- | ---------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | yes | The name of the managed license | -| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". "blacklisted" and "approved" are deprecated. | +| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". | ```shell curl --data "name=MIT&approval_status=denied" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/managed_licenses" @@ -93,7 +93,7 @@ Example response: { "id": 1, "name": "MIT", - "approval_status": "approved" + "approval_status": "allowed" } ``` @@ -128,7 +128,7 @@ PATCH /projects/:id/managed_licenses/:managed_license_id | --------------- | ------- | --------------------------------- | ------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `managed_license_id` | integer/string | yes | The ID or URL-encoded name of the license belonging to the project | -| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". "blacklisted" and "approved" are deprecated. | +| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". | ```shell curl --request PATCH --data "approval_status=denied" \ @@ -141,6 +141,6 @@ Example response: { "id": 1, "name": "MIT", - "approval_status": "blacklisted" + "approval_status": "denied" } ``` diff --git a/doc/api/members.md b/doc/api/members.md index 77a91436e6c..1db9714bfd1 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -367,6 +367,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "last_activity_on": "2021-01-27", "membership_type": "group_member", + "membership_state": "active", "removable": true, "created_at": "2021-01-03T12:16:02.000Z" }, @@ -380,6 +381,7 @@ Example response: "email": "john@example.com", "last_activity_on": "2021-01-25", "membership_type": "group_member", + "membership_state": "active", "removable": true, "created_at": "2021-01-04T18:46:42.000Z" }, @@ -392,6 +394,7 @@ Example response: "web_url": "http://192.168.1.8:3000/root", "last_activity_on": "2021-01-20", "membership_type": "group_invite", + "membership_state": "awaiting", "removable": false, "created_at": "2021-01-09T07:12:31.000Z" } @@ -480,6 +483,35 @@ DELETE /groups/:id/billable_members/:user_id curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/billable_members/:user_id" ``` +## Change membership state of a user in a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86705) in GitLab 15.0. + +Changes the membership state of a user in a group. The state is applied to +all subgroups and projects. + +```plaintext +PUT /groups/:id/members/:user_id/state +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `user_id` | integer | yes | The user ID of the member. | +| `state` | string | yes | The new state for the user. State is either `awaiting` or `active`. | + +```shell +curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/members/:user_id/state?state=active" +``` + +Example response: + +```json +{ + "success":true +} +``` + ## Add a member to a group or project Adds a member to a group or project. diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 721e4db3314..37a926366df 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -277,12 +277,7 @@ GET /projects/:id/approval_rules/:approval_rule_id > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. - -WARNING: -The Vulnerability-Check feature, including the Vulnerability-Check attributes listed here, is in its -end-of-life process. It is [deprecated](../update/deprecations.md#vulnerability-check) -in GitLab 14.8, and is planned for removal in GitLab 15.0. Users should migrate to the new -[Security Approval Policies](../user/application_security/policies/#scan-result-policy-editor). +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. You can create project approval rules using the following endpoint: @@ -301,11 +296,7 @@ POST /projects/:id/approval_rules | `user_ids` | Array | no | The ids of users as approvers | | `group_ids` | Array | no | The ids of groups as approvers | | `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `vulnerability`, `license_scanning`, `code_coverage`. The `vulnerability` report type is part of the Vulnerability-Check feature, which deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `scanners` | Array | no | The security scanners the Vulnerability-Check approval rule considers. The supported scanners are: `sast`, `secret_detection`, `dependency_scanning`, `container_scanning`, `dast`, `coverage_fuzzing`, `api_fuzzing`. Defaults to all supported scanners. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `severity_levels` | Array | no | The severity levels the Vulnerability-Check approval rule considers. The supported severity levels are: `info`, `unknown`, `low`, `medium`, `high`, `critical`. Defaults to `unknown`, `high`, and `critical`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `vulnerabilities_allowed` | integer | no | The number of vulnerabilities allowed for the Vulnerability-Check approval rule. Defaults to `0`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `vulnerability_states` | Array | no | The vulnerability states the Vulnerability-Check approval rule considers. The supported vulnerability states are: `newly_detected` (default), `detected`, `confirmed`, `resolved`, `dismissed`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | +| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `license_scanning` and `code_coverage`.| ```json { @@ -409,12 +400,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. - -WARNING: -The Vulnerability-Check feature, including the Vulnerability-Check attributes listed here, is in its -end-of-life process. It is [deprecated](../update/deprecations.md#vulnerability-check) -in GitLab 14.8, and is planned for removal in GitLab 15.0. Users should migrate to the new -[Security Approval Policies](../user/application_security/policies/#scan-result-policy-editor). +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. You can update project approval rules using the following endpoint: @@ -435,10 +421,6 @@ PUT /projects/:id/approval_rules/:approval_rule_id | `user_ids` | Array | no | The ids of users as approvers | | `group_ids` | Array | no | The ids of groups as approvers | | `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `scanners` | Array | no | The security scanners the Vulnerability-Check approval rule considers. The supported scanners are: `sast`, `secret_detection`, `dependency_scanning`, `container_scanning`, `dast`, `coverage_fuzzing`, `api_fuzzing`. Defaults to all supported scanners. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `severity_levels` | Array | no | The severity levels the Vulnerability-Check approval rule considers. The supported severity levels are: `info`, `unknown`, `low`, `medium`, `high`, `critical`. Defaults to `unknown`, `high`, and `critical`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `vulnerabilities_allowed` | integer | no | The number of vulnerabilities allowed for the Vulnerability-Check approval rule. Defaults to `0`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | -| `vulnerability_states` | Array | no | The vulnerability states the Vulnerability-Check approval rule considers. The supported vulnerability states are: `newly_detected` (default), `detected`, `confirmed`, `resolved`, `dismissed`. Deprecated in GitLab 14.8, and planned for removal in GitLab 15.0. | ```json { diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 0c065c0f2f5..abe9cb65f95 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -1103,8 +1103,8 @@ POST /projects/:id/merge_requests | `target_branch` | string | yes | The target branch. | | `title` | string | yes | Title of MR. | | `assignee_id` | integer | no | Assignee user ID. | -| `assignee_ids` | integer array | no | The ID of the user(s) to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | -| `reviewer_ids` | integer array | no | The ID of the user(s) added as a reviewer to the MR. If set to `0` or left empty, no reviewers are added. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | +| `assignee_ids` | integer array | no | The ID of the users to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | +| `reviewer_ids` | integer array | no | The ID of the users added as a reviewer to the MR. If set to `0` or left empty, no reviewers are added. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | | `description` | string | no | Description of MR. Limited to 1,048,576 characters. | | `target_project_id` | integer | no | The target project (numeric ID). | | `labels` | string | no | Labels for MR as a comma-separated list. | @@ -1271,8 +1271,8 @@ PUT /projects/:id/merge_requests/:merge_request_iid | `target_branch` | string | no | The target branch. | | `title` | string | no | Title of MR. | | `assignee_id` | integer | no | The ID of the user to assign the merge request to. Set to `0` or provide an empty value to unassign all assignees. | -| `assignee_ids` | integer array | no | The ID of the user(s) to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | -| `reviewer_ids` | integer array | no | The ID of the user(s) set as a reviewer to the MR. Set the value to `0` or provide an empty value to unset all reviewers. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | +| `assignee_ids` | integer array | no | The ID of the users to assign the MR to. Set to `0` or provide an empty value to unassign all assignees. | +| `reviewer_ids` | integer array | no | The ID of the users set as a reviewer to the MR. Set the value to `0` or provide an empty value to unset all reviewers. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | | `milestone_id` | integer | no | The global ID of a milestone to assign the merge request to. Set to `0` or provide an empty value to unassign a milestone.| | `labels` | string | no | Comma-separated label names for a merge request. Set to an empty string to unassign all labels. | | `add_labels` | string | no | Comma-separated label names to add to a merge request. | diff --git a/doc/api/milestones.md b/doc/api/milestones.md index 3c1e09eaace..48b6143a020 100644 --- a/doc/api/milestones.md +++ b/doc/api/milestones.md @@ -113,7 +113,9 @@ Parameters: ## Delete project milestone -Only for users with the Developer role in the project. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. + +Only for users with at least the Reporter role in the project. ```plaintext DELETE /projects/:id/milestones/:milestone_id @@ -158,9 +160,9 @@ Parameters: ## Promote project milestone to a group milestone -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53861) in GitLab 11.9 +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. -Only for users with the Developer role in the group. +Only for users with at least the Reporter role in the group. ```plaintext POST /projects/:id/milestones/:milestone_id/promote diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index ad93d8033d0..aa9a86f33d5 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -33,7 +33,7 @@ Implicit grant and Resource Owner Password Credentials flows. Refer to the [OAuth RFC](https://tools.ietf.org/html/rfc6749) to find out how all those flows work and pick the right one for your use case. -Both **authorization code** (with or without PKCE) and **implicit grant** flows require `application` to be +Authorization code (with or without PKCE) flow requires `application` to be registered first via the `/profile/applications` page in your user's account. During registration, by enabling proper scopes, you can limit the range of resources which the `application` can access. Upon creation, you obtain the @@ -59,8 +59,6 @@ For development, GitLab allows insecure HTTP redirect URIs. As OAuth 2.0 bases its security entirely on the transport layer, you should not use unprotected URIs. For more information, see the [OAuth 2.0 RFC](https://tools.ietf.org/html/rfc6749#section-3.1.2.1) and the [OAuth 2.0 Threat Model RFC](https://tools.ietf.org/html/rfc6819#section-4.4.2.1). -These factors are particularly important when using the -[Implicit grant flow](#implicit-grant-flow-deprecated), where actual credentials are included in the `redirect_uri`. In the following sections you can find detailed instructions on how to obtain authorization with each flow. @@ -319,12 +317,13 @@ access_token = client.password.get_token('user@example.com', 'secret') puts access_token.token ``` -### Implicit grant flow (DEPRECATED) +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> + +### Implicit grant flow (removed) -WARNING: Implicit grant flow is inherently insecure and the IETF has removed it in [OAuth 2.1](https://oauth.net/2.1/). -It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/288516) in GitLab 14.0, and is planned for -[removal](https://gitlab.com/gitlab-org/gitlab/-/issues/344609) in GitLab 15.0. +It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/288516) in GitLab 14.0 and is +[removed](https://gitlab.com/gitlab-org/gitlab/-/issues/344609) in GitLab 15.0. We recommend that you use [Authorization code with PKCE](#authorization-code-with-proof-key-for-code-exchange-pkce) instead. @@ -353,6 +352,8 @@ parameters, for example: https://example.com/oauth/redirect#access_token=ABCDExyz123&state=YOUR_UNIQUE_STATE_HASH&token_type=bearer&expires_in=3600 ``` +<!--- end_remove --> + ## Access GitLab API with `access token` The `access token` allows you to make requests to the API on behalf of a user. diff --git a/doc/api/packages.md b/doc/api/packages.md index 555b8d5e054..9d9c21546cd 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -14,6 +14,8 @@ This is the API documentation of [GitLab Packages](../administration/packages/in Get a list of project packages. All package types are included in results. When accessed without authentication, only packages of public projects are returned. +By default, packages with `default` and `error` status are returned. Use the `status` parameter to view other +packages. ```plaintext GET /projects/:id/packages @@ -27,7 +29,7 @@ GET /projects/:id/packages | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, `terraform_module`, or `golang`. (_Introduced in GitLab 12.9_) | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_Introduced in GitLab 12.9_) | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. (_Introduced in GitLab 13.8_) -| `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, or `processing`. (_Introduced in GitLab 13.9_) +| `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, `processing`, `error`, or `pending_destruction`. (_Introduced in GitLab 13.9_) ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/packages" @@ -78,6 +80,8 @@ can result in malformed data or broken packages. Get a list of project packages at the group level. When accessed without authentication, only packages of public projects are returned. +By default, packages with `default` and `error` status are returned. Use the `status` parameter to view other +packages. ```plaintext GET /groups/:id/packages @@ -92,7 +96,7 @@ GET /groups/:id/packages | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, or `golang`. (_Introduced in GitLab 12.9_) | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30980) in GitLab 13.0_) | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. (_Introduced in GitLab 13.8_) -| `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, or `processing`. (_Introduced in GitLab 13.9_) +| `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, `processing`, `error`, or `pending_destruction`. (_Introduced in GitLab 13.9_) ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/:id/packages?exclude_subgroups=false" diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index 610fd97c810..1c2cfef8e1b 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Pages domains API **(FREE)** -Endpoints for connecting custom domain(s) and TLS certificates in [GitLab Pages](https://about.gitlab.com/stages-devops-lifecycle/pages/). +Endpoints for connecting custom domains and TLS certificates in [GitLab Pages](https://about.gitlab.com/stages-devops-lifecycle/pages/). The GitLab Pages feature must be enabled to use these endpoints. Find out more about [administering](../administration/pages/index.md) and [using](../user/project/pages/index.md) the feature. diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index b51866fe9b1..c49af2a745c 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -72,10 +72,17 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a ## Revoke a personal access token +Revoke a personal access token by either: + +- Using the ID of the personal access token. +- Passing it to the API in a header. + +### Using a personal access token ID + > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216004) in GitLab 13.3. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/270200) from GitLab Ultimate to GitLab Free in 13.6. -Revoke a personal access token. +Revoke a personal access token using its ID. ```plaintext DELETE /personal_access_tokens/:id @@ -92,10 +99,29 @@ Non-administrators can revoke their own tokens. Administrators can revoke tokens curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/<personal_access_token_id>" ``` -### Responses +#### Responses + +- `204: No Content` if successfully revoked. +- `400: Bad Request` if not revoked successfully. + +### Using a request header + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350240) in GitLab 15.0. + +Revokes a personal access token that is passed in using a request header. + +```plaintext +DELETE /personal_access_tokens/self +``` + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/self" +``` + +#### Responses - `204: No Content` if successfully revoked. -- `400 Bad Request` if not revoked successfully. +- `400: Bad Request` if not revoked successfully. ## Create a personal access token (administrator only) diff --git a/doc/api/plan_limits.md b/doc/api/plan_limits.md index 75c8d241513..0b6f89675d0 100644 --- a/doc/api/plan_limits.md +++ b/doc/api/plan_limits.md @@ -35,6 +35,14 @@ Example response: ```json { + "ci_pipeline_size": 0, + "ci_active_jobs": 0, + "ci_active_pipelines": 0, + "ci_project_subscriptions": 2, + "ci_pipeline_schedules": 10, + "ci_needs_size_limit": 50, + "ci_registered_group_runners": 1000, + "ci_registered_project_runners": 1000, "conan_max_file_size": 3221225472, "generic_packages_max_file_size": 5368709120, "helm_max_file_size": 5242880, @@ -42,7 +50,8 @@ Example response: "npm_max_file_size": 524288000, "nuget_max_file_size": 524288000, "pypi_max_file_size": 3221225472, - "terraform_module_max_file_size": 1073741824 + "terraform_module_max_file_size": 1073741824, + "storage_size_limit": 15000 } ``` @@ -57,6 +66,14 @@ PUT /application/plan_limits | Attribute | Type | Required | Description | | --------------------------------- | ------- | -------- | ----------- | | `plan_name` | string | yes | Name of the plan to update. | +| `ci_pipeline_size` | integer | no | Maximum number of jobs in a single pipeline. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_active_jobs` | integer | no | Total number of jobs in currently active pipelines. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_active_pipelines` | integer | no | Maximum number of active pipelines per project. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_project_subscriptions` | integer | no | Maximum number of pipeline subscriptions to and from a project. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_pipeline_schedules` | integer | no | Maximum number of pipeline schedules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_needs_size_limit` | integer | no | Maximum number of [DAG](../ci/directed_acyclic_graph/index.md) dependencies that a job can have. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_registered_group_runners` | integer | no | Maximum number of runners registered per group. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | +| `ci_registered_project_runners` | integer | no | Maximum number of runners registered per project. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | | `conan_max_file_size` | integer | no | Maximum Conan package file size in bytes. | | `generic_packages_max_file_size` | integer | no | Maximum generic package file size in bytes. | | `helm_max_file_size` | integer | no | Maximum Helm chart file size in bytes. | @@ -65,6 +82,7 @@ PUT /application/plan_limits | `nuget_max_file_size` | integer | no | Maximum NuGet package file size in bytes. | | `pypi_max_file_size` | integer | no | Maximum PyPI package file size in bytes. | | `terraform_module_max_file_size` | integer | no | Maximum Terraform Module package file size in bytes. | +| `storage_size_limit` | integer | no | Maximum storage size for the root namespace in megabytes. | ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/application/plan_limits?plan_name=default&conan_max_file_size=3221225472" @@ -74,6 +92,14 @@ Example response: ```json { + "ci_pipeline_size": 0, + "ci_active_jobs": 0, + "ci_active_pipelines": 0, + "ci_project_subscriptions": 2, + "ci_pipeline_schedules": 10, + "ci_needs_size_limit": 50, + "ci_registered_group_runners": 1000, + "ci_registered_project_runners": 1000, "conan_max_file_size": 3221225472, "generic_packages_max_file_size": 5368709120, "helm_max_file_size": 5242880, diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 437522b0946..5525000e336 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -302,7 +302,7 @@ Parameters: NOTE: `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added -through the ["Add existing Kubernetes cluster"](../user/project/clusters/add_remove_clusters.md#add-existing-cluster) option or +through the ["Add existing Kubernetes cluster"](../user/project/clusters/add_existing_cluster.md) option or through the ["Add existing cluster to project"](#add-existing-cluster-to-project) endpoint. Example request: diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index b9ebfa0fc5c..3f2cc09aa1e 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -49,6 +49,12 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla NOTE: The upload request is sent with `Content-Type: application/gzip` header. Ensure that your pre-signed URL includes this as part of the signature. +NOTE: +As an administrator, you can modify the maximum export file size. By default, +it is set to `0`, for unlimited. To change this value, edit `max_export_size` +in the [Application settings API](settings.md#change-application-settings) +or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md). + ## Export status Get the status of export. @@ -184,7 +190,7 @@ requests.post(url, headers=headers, data=data, files=files) NOTE: The maximum import file size can be set by the Administrator, default is `0` (unlimited).. -As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. ## Import a file from a remote object storage diff --git a/doc/api/projects.md b/doc/api/projects.md index c40d39ee981..64a2e0d801b 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -36,6 +36,8 @@ There are three options for `merge_method` to choose from: ## List all projects +> The `_links.cluster_agents` attribute in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 15.0. + Get a list of all visible projects across GitLab for the authenticated user. When accessed without authentication, only public projects with _simple_ fields are returned. @@ -47,6 +49,7 @@ GET /projects | Attribute | Type | Required | Description | |--------------------------------------------|----------|------------------------|-------------| | `archived` | boolean | **{dotted-circle}** No | Limit by archived status. | +| `build_coverage_regex` | string | **{dotted-circle}** No | Test coverage parsing. (`deprecated`, it is [scheduled to be removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357401)) | | `id_after` | integer | **{dotted-circle}** No | Limit results to projects with IDs greater than the specified ID. | | `id_before` | integer | **{dotted-circle}** No | Limit results to projects with IDs less than the specified ID. | | `imported` | boolean | **{dotted-circle}** No | Limit results to projects which were imported from external systems by current user. | @@ -208,7 +211,8 @@ When the user is authenticated and `simple` is not set this returns something li "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } }, { @@ -325,7 +329,8 @@ When the user is authenticated and `simple` is not set this returns something li "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ] @@ -371,6 +376,8 @@ Keyset pagination supports only `order_by=id`. Other sorting options aren't avai ## List user projects +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Get a list of visible projects owned by the given user. When accessed without authentication, only public projects are returned. @@ -499,7 +506,8 @@ GET /users/:user_id/projects "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } }, { @@ -616,7 +624,8 @@ GET /users/:user_id/projects "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ] @@ -624,6 +633,8 @@ GET /users/:user_id/projects ## List projects starred by a user +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Get a list of visible projects starred by the given user. When accessed without authentication, only public projects are returned. @@ -744,7 +755,8 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } }, { @@ -858,7 +870,8 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ] @@ -866,6 +879,8 @@ Example response: ## Get single project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Get a specific project. This endpoint can be accessed without authentication if the project is publicly accessible. @@ -1032,7 +1047,8 @@ GET /projects/:id "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ``` @@ -1212,6 +1228,16 @@ where `password` is a public access key with the `api` scope enabled. POST /projects ``` +Example request: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your-token>" \ + --header "Content-Type: application/json" --data '{ + "name": "new_project", "description": "New Project", "path": "new_project", + "namespace_id": "42", "initialize_with_readme": "true"}' \ + --url 'https://gitlab.example.com/api/v4/projects/' +``` + | Attribute | Type | Required | Description | |-------------------------------------------------------------|---------|------------------------|-------------| | `name` | string | **{check-circle}** Yes (if path isn't provided) | The name of the new project. Equals path if not provided. | @@ -1224,7 +1250,6 @@ POST /projects | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | | `avatar` | mixed | **{dotted-circle}** No | Image file for avatar of the project. | -| `build_coverage_regex` | string | **{dotted-circle}** No | Test coverage parsing. | | `build_git_strategy` | string | **{dotted-circle}** No | The Git strategy. Defaults to `fetch`. | | `build_timeout` | integer | **{dotted-circle}** No | The maximum amount of time, in seconds, that a job can run. | | `builds_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1238,8 +1263,8 @@ POST /projects | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | | `forking_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `group_with_project_templates_id` **(PREMIUM)** | integer | **{dotted-circle}** No | For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true. | -| `import_url` | string | **{dotted-circle}** No | URL to import repository from. | -| `initialize_with_readme` | boolean | **{dotted-circle}** No | `false` by default. | +| `import_url` | string | **{dotted-circle}** No | URL to import repository from. When this isn't empty, you must not set `initialize_with_readme` to `true`. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | +| `initialize_with_readme` | boolean | **{dotted-circle}** No | Whether to create a Git repository with just a `README.md` file. Default is `false`. When this is true, you must not pass `import_url` or other attributes of this endpoint which specify alternative contents for the repository. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | | `issues_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `issues_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable issues for this project. Use `issues_access_level` instead. | | `jobs_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable jobs for this project. Use `builds_access_level` instead. | @@ -1303,7 +1328,6 @@ POST /projects/user/:user_id | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | | `avatar` | mixed | **{dotted-circle}** No | Image file for avatar of the project. | -| `build_coverage_regex` | string | **{dotted-circle}** No | Test coverage parsing. | | `build_git_strategy` | string | **{dotted-circle}** No | The Git strategy. Defaults to `fetch`. | | `build_timeout` | integer | **{dotted-circle}** No | The maximum amount of time, in seconds, that a job can run. | | `builds_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1392,7 +1416,6 @@ Supported attributes: | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | | `avatar` | mixed | **{dotted-circle}** No | Image file for avatar of the project. | -| `build_coverage_regex` | string | **{dotted-circle}** No | Test coverage parsing. | | `build_git_strategy` | string | **{dotted-circle}** No | The Git strategy. Defaults to `fetch`. | | `build_timeout` | integer | **{dotted-circle}** No | The maximum amount of time, in seconds, that a job can run. | | `builds_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | @@ -1407,7 +1430,7 @@ Supported attributes: | `emails_disabled` | boolean | **{dotted-circle}** No | Disable email notifications. | | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | | `forking_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | -| `import_url` | string | **{dotted-circle}** No | URL to import repository from. | +| `import_url` | string | **{dotted-circle}** No | URL the repository was imported from. | | `issues_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `issues_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable issues for this project. Use `issues_access_level` instead. | | `issues_template` **(PREMIUM)** | string | **{dotted-circle}** No | Default description for Issues. Description is parsed with GitLab Flavored Markdown. See [Templates for issues and merge requests](#templates-for-issues-and-merge-requests). | @@ -1483,6 +1506,8 @@ POST /projects/:id/fork ## List forks of a project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + List the projects accessible to the calling user that have an established, forked relationship with the specified project @@ -1585,7 +1610,8 @@ Example responses: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ] @@ -1593,6 +1619,8 @@ Example responses: ## Star a project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Stars a given project. Returns status code `304` if the project is already starred. @@ -1688,13 +1716,16 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ``` ## Unstar a project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Unstars a given project. Returns status code `304` if the project is not starred. ```plaintext @@ -1789,7 +1820,8 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ``` @@ -1869,6 +1901,8 @@ Example response: ## Archive a project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Archives the project if the user is either an administrator or the owner of this project. This action is idempotent, thus archiving an already archived project does not change the project. @@ -1984,13 +2018,16 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ``` ## Unarchive a project +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + Unarchives the project if the user is either an administrator or the owner of this project. This action is idempotent, thus unarchiving a non-archived project doesn't change the project. @@ -2106,7 +2143,8 @@ Example response: "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", "labels": "http://example.com/api/v4/projects/1/labels", "events": "http://example.com/api/v4/projects/1/events", - "members": "http://example.com/api/v4/projects/1/members" + "members": "http://example.com/api/v4/projects/1/members", + "cluster_agents": "http://example.com/api/v4/projects/1/cluster_agents" } } ``` @@ -2507,7 +2545,7 @@ POST /projects/:id/housekeeping ### Get project push rules -Get the [push rules](../user/project/repository/push_rules.md#enabling-push-rules) of a +Get the [push rules](../user/project/repository/push_rules.md) of a project. ```plaintext @@ -2614,6 +2652,8 @@ DELETE /projects/:id/push_rule ## Transfer a project to a new namespace +> The `_links.cluster_agents` attribute in the response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347047) in GitLab 14.10. + See the [Project documentation](../user/project/settings/index.md#transferring-an-existing-project-into-another-namespace) for prerequisites to transfer a project. @@ -2718,7 +2758,7 @@ Example response: "public_jobs": true, "build_timeout": 3600, "auto_cancel_pending_pipelines": "enabled", - "build_coverage_regex": null, + "build_coverage_regex": null, // deprecated, it is scheduled to be removed https://gitlab.com/gitlab-org/gitlab/-/issues/357401 "ci_config_path": null, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index fc7eb5caf6d..fa099afd680 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -20,6 +20,16 @@ Currently, these levels are recognized: 60 => Admin access ``` +## Group inheritance types + +Group inheritance allows deploy access levels and access rules to take inherited group membership into account. The group inheritance types are defined by `ProtectedEnvironments::Authorizable::GROUP_INHERITANCE_TYPE`. +The following types are recognized: + +```plaintext +0 => Direct group membership only (default) +1 => All inherited groups +``` + ## List protected environments Gets a list of protected environments from a project: @@ -47,7 +57,8 @@ Example response: "access_level":40, "access_level_description":"Maintainers", "user_id":null, - "group_id":null + "group_id":null, + "group_inheritance_type": 0 } ], "required_approval_count": 0 @@ -82,7 +93,8 @@ Example response: "access_level": 40, "access_level_description": "Maintainers", "user_id": null, - "group_id": null + "group_id": null, + "group_inheritance_type": 0 } ], "required_approval_count": 0 @@ -114,7 +126,8 @@ curl --header 'Content-Type: application/json' --request POST \ Elements in the `deploy_access_levels` and `approval_rules` array should be one of `user_id`, `group_id` or `access_level`, and take the form `{user_id: integer}`, `{group_id: integer}` or -`{access_level: integer}`. +`{access_level: integer}`. Optionally you can specify the `group_inheritance_type` on each as one of the [valid group inheritance types](#group-inheritance-types). + Each user must have access to the project and each group must [have this project shared](../user/project/members/share_project_with_groups.md). Example response: @@ -127,7 +140,8 @@ Example response: "access_level": 40, "access_level_description": "protected-access-group", "user_id": null, - "group_id": 9899826 + "group_id": 9899826, + "group_inheritance_type": 0 } ], "required_approval_count": 0, @@ -137,14 +151,16 @@ Example response: "group_id": 134, "access_level": null, "access_level_description": "qa-group", - "required_approvals": 1 + "required_approvals": 1, + "group_inheritance_type": 0 }, { "user_id": null, "group_id": 135, "access_level": null, "access_level_description": "security-group", - "required_approvals": 2 + "required_approvals": 2, + "group_inheritance_type": 0 } ] } diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 1fdc48a6d92..8097fecea87 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -100,11 +100,14 @@ Allows you to receive blame information. Each blame range contains lines and cor GET /projects/:id/repository/files/:file_path/blame ``` -| Attribute | Type | Required | Description | -|-------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | -| `file_path` | string | yes | URL encoded full path to new file. Ex. `lib%2Fclass%2Erb`. | -| `ref` | string | yes | The name of branch, tag or commit | +| Attribute | Type | Required | Description | +|-----------------|-------------------|----------|--------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `file_path` | string | yes | URL encoded full path to new file. Ex. `lib%2Fclass%2Erb`. | +| `ref` | string | yes | The name of branch, tag or commit | +| `range` | hash | no | Blame range | +| `range[start]` | integer | yes | The first line of the range to blame | +| `range[end]` | integer | yes | The last line of the range to blame | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master" @@ -163,6 +166,41 @@ X-Gitlab-Execute-Filemode: false ... ``` +### Examples + +To request a blame range, specify `range[start]` and `range[end]` parameters with the start and end line numbers of the file. + +```shell +curl --head --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/13083/repository/files/path%2Fto%2Ffile.rb/blame?ref=master&range[start]=1&range[end]=2" +``` + +Example response: + +```json +[ + { + "commit": { + "id": "d42409d56517157c48bf3bd97d3f75974dde19fb", + "message": "Add feature\n\nalso fix bug\n", + "parent_ids": [ + "cc6e14f9328fa6d7b5a0d3c30dc2002a3f2a3822" + ], + "authored_date": "2015-12-18T08:12:22.000Z", + "author_name": "John Doe", + "author_email": "john.doe@example.com", + "committed_date": "2015-12-18T08:12:22.000Z", + "committer_name": "John Doe", + "committer_email": "john.doe@example.com" + }, + "lines": [ + "require 'fileutils'", + "require 'open3'" + ] + }, + ... +] +``` + ## Get raw file from repository ```plaintext diff --git a/doc/api/resource_access_tokens.md b/doc/api/resource_access_tokens.md deleted file mode 100644 index a9c43625193..00000000000 --- a/doc/api/resource_access_tokens.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'project_access_tokens.md' -remove_date: '2022-04-06' ---- - -This document was moved to [another location](project_access_tokens.md). - -<!-- This redirect file can be deleted after <2022-04-06>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/api/resource_groups.md b/doc/api/resource_groups.md index 237ba6e7d72..f70ad2f09e7 100644 --- a/doc/api/resource_groups.md +++ b/doc/api/resource_groups.md @@ -8,6 +8,34 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can read more about [controlling the job concurrency with resource groups](../ci/resource_groups/index.md). +## Get all resource groups for a project + +```plaintext +GET /projects/:id/resource_groups +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/resource_groups" +``` + +Example of response + +```json +[ + { + "id": 3, + "key": "production", + "process_mode": "unordered", + "created_at": "2021-09-01T08:04:59.650Z", + "updated_at": "2021-09-01T08:04:59.650Z" + } +] +``` + ## Get a specific resource group ```plaintext diff --git a/doc/api/runners.md b/doc/api/runners.md index 304f2494f70..7519c3595b6 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -44,13 +44,13 @@ GET /runners?paused=true GET /runners?tag_list=tag1,tag2 ``` -| Attribute | Type | Required | Description | -|------------|--------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `scope` | string | no | Deprecated: Use `type` or `status` instead. The scope of specific runners to show, one of: `active`, `paused`, `online` and `offline`; showing all runners if none provided | -| `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | -| `status` | string | no | The status of runners to show, one of: `online`, `offline` and `not_connected`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | -| `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | -| `tag_list` | string array | no | List of the runner's tags | +| Attribute | Type | Required | Description | +|------------|--------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `scope` | string | no | Deprecated: Use `type` or `status` instead. The scope of specific runners to show, one of: `active`, `paused`, `online` and `offline`; showing all runners if none provided | +| `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | +| `status` | string | no | The status of runners to show, one of: `online`, `offline` and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | +| `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | +| `tag_list` | string array | no | List of the runner's tags | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners" @@ -610,7 +610,7 @@ Example response: "runner_type": "instance_type", "name": "gitlab-runner", "online": null, - "status": "not_connected" + "status": "never_contacted" }, { "id": 6, @@ -634,7 +634,7 @@ Example response: "runner_type": "group_type", "name": "gitlab-runner", "online": null, - "status": "not_connected" + "status": "never_contacted" } ] ``` @@ -651,7 +651,7 @@ POST /runners |--------------------|--------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| | `token` | string | yes | [Registration token](#registration-and-authentication-tokens) | | `description` | string | no | Runner's description | -| `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version` is displayed in the Admin area of the UI | +| `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version` is displayed in the Admin Area of the UI | | `active` | boolean | no | Deprecated: Use `:paused` instead. Whether the runner is allowed to receive jobs | | `paused` | boolean | no | Whether the runner should ignore new jobs | | `locked` | boolean | no | Whether the runner should be locked for current project | diff --git a/doc/api/secure_files.md b/doc/api/secure_files.md index 7d30cc0c4c7..3532b5a7aeb 100644 --- a/doc/api/secure_files.md +++ b/doc/api/secure_files.md @@ -11,7 +11,7 @@ type: reference, api FLAG: On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ci_secure_files`. Limited to 100 secure files per project. Files must be smaller than 5 MB. The feature is not ready for production use. +ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ci_secure_files`. Limited to 100 secure files per project. Files must be smaller than 5 MB. The feature is not ready for production use. ## List project secure files @@ -42,7 +42,6 @@ Example response: "name": "myfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", "checksum_algorithm": "sha256", - "permissions": "read_only", "created_at": "2022-02-22T22:22:22.222Z" }, { @@ -50,7 +49,6 @@ Example response: "name": "myotherfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aa2", "checksum_algorithm": "sha256", - "permissions": "execute", "created_at": "2022-02-22T22:22:22.222Z" } ] @@ -85,7 +83,6 @@ Example response: "name": "myfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", "checksum_algorithm": "sha256", - "permissions": "read_only", "created_at": "2022-02-22T22:22:22.222Z" } ``` @@ -105,7 +102,6 @@ Supported attributes: | `project_id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | | `name` | string | **{check-circle}** Yes | The `name` of the file being uploaded. The file name must be unique within the project. | | `file` | file | **{check-circle}** Yes | The `file` being uploaded (5 MB limit). | -| `permissions` | string | **{dotted-circle}** No | The file is created with the specified permissions when created in the CI/CD job. Available types are: `read_only` (default), `read_write`, and `execute`. | Example request: @@ -122,7 +118,6 @@ Example response: "name": "myfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", "checksum_algorithm": "sha256", - "permissions": "read_only", "created_at": "2022-02-22T22:22:22.222Z" } ``` diff --git a/doc/api/settings.md b/doc/api/settings.md index cc9df7917e2..193f18779f5 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -36,6 +36,7 @@ Example response: "password_authentication_enabled_for_web" : true, "after_sign_out_path" : null, "max_attachment_size" : 10, + "max_export_size": 50, "max_import_size": 50, "user_oauth_applications" : true, "updated_at" : "2016-01-04T15:44:55.176Z", @@ -103,7 +104,7 @@ Example response: ``` Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) may also see -the `file_template_project_id`, `delayed_project_deletion`, `deletion_adjourned_period`, or the `geo_node_allowed_ips` parameters: +the `file_template_project_id`, `delayed_project_deletion`, `delayed_group_deletion`, `deletion_adjourned_period`, or the `geo_node_allowed_ips` parameters: ```json { @@ -112,6 +113,7 @@ the `file_template_project_id`, `delayed_project_deletion`, `deletion_adjourned_ "file_template_project_id": 1, "geo_node_allowed_ips": "0.0.0.0/0, ::/0", "delayed_project_deletion": false, + "delayed_group_deletion": false, "deletion_adjourned_period": 7, ... } @@ -146,6 +148,7 @@ Example response: "default_branch_protection": 2, "restricted_visibility_levels": [], "max_attachment_size": 10, + "max_export_size": 50, "max_import_size": 50, "session_expire_delay": 10080, "default_ci_config_path" : null, @@ -216,7 +219,8 @@ these parameters: - `file_template_project_id` - `geo_node_allowed_ips` - `geo_status_timeout` -- `delayed_project_delection` +- `delayed_project_deletion` +- `delayed_group_deletion` - `deletion_adjourned_period` Example responses: **(PREMIUM SELF)** @@ -249,8 +253,8 @@ listed in the descriptions of the relevant settings. | `asset_proxy_enabled` | boolean | no | (**If enabled, requires:** `asset_proxy_url`) Enable proxying of assets. GitLab restart is required to apply changes. | | `asset_proxy_secret_key` | string | no | Shared secret with the asset proxy server. GitLab restart is required to apply changes. | | `asset_proxy_url` | string | no | URL of the asset proxy server. GitLab restart is required to apply changes. | -| `asset_proxy_whitelist` | string or array of strings | no | (Deprecated: Use `asset_proxy_allowlist` instead) Assets that match these domain(s) are **not** proxied. Wildcards allowed. Your GitLab installation URL is automatically allowlisted. GitLab restart is required to apply changes. | -| `asset_proxy_allowlist` | string or array of strings | no | Assets that match these domain(s) are **not** proxied. Wildcards allowed. Your GitLab installation URL is automatically allowlisted. GitLab restart is required to apply changes. | +| `asset_proxy_whitelist` | string or array of strings | no | (Deprecated: Use `asset_proxy_allowlist` instead) Assets that match these domains are **not** proxied. Wildcards allowed. Your GitLab installation URL is automatically allowlisted. GitLab restart is required to apply changes. | +| `asset_proxy_allowlist` | string or array of strings | no | Assets that match these domains are **not** proxied. Wildcards allowed. Your GitLab installation URL is automatically allowlisted. GitLab restart is required to apply changes. | | `authorized_keys_enabled` | boolean | no | By default, we write to the `authorized_keys` file to support Git over SSH without additional configuration. GitLab can be optimized to authenticate SSH keys via the database file. Only disable this if you have configured your OpenSSH server to use the AuthorizedKeysCommand. | | `auto_devops_domain` | string | no | Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages. | | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for projects by default. It automatically builds, tests, and deploys applications based on a predefined CI/CD configuration. | @@ -274,8 +278,9 @@ listed in the descriptions of the relevant settings. | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. | -| `delete_inactive_projects` | boolean | no | Enable inactive project deletion feature. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. | -| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. +| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion by default in new groups. Requires both `delayed_group_deletion` to be true and `deletion_adjourned_period` to be greater than 0. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. | +| `delete_inactive_projects` | boolean | no | Enable inactive project deletion feature. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0 (with feature flag `inactive_projects_deletion`, disabled by default). | +| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. On every update, a hook on `deletion_adjourned_period` sets the value of `delayed_group_deletion` to `true` if `deletion_adjourned_period` is greater than 0 and `false` if `deletion_adjourned_period` is 0. | `diff_max_patch_bytes` | integer | no | Maximum [diff patch size](../user/admin_area/diff_limits.md), in bytes. | | `diff_max_files` | integer | no | Maximum [files in a diff](../user/admin_area/diff_limits.md). | | `diff_max_lines` | integer | no | Maximum [lines in a diff](../user/admin_area/diff_limits.md). | @@ -283,7 +288,7 @@ listed in the descriptions of the relevant settings. | `disabled_oauth_sign_in_sources` | array of strings | no | Disabled OAuth sign-in sources. | | `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS rebinding attack protection. | | `domain_denylist_enabled` | boolean | no | (**If enabled, requires:** `domain_denylist`) Allows blocking sign-ups from emails from specific domains. | -| `domain_denylist` | array of strings | no | Users with email addresses that match these domain(s) **cannot** sign up. Wildcards allowed. Use separate lines for multiple entries. Ex: `domain.com`, `*.domain.com`. | +| `domain_denylist` | array of strings | no | Users with email addresses that match these domains **cannot** sign up. Wildcards allowed. Use separate lines for multiple entries. Ex: `domain.com`, `*.domain.com`. | | `domain_allowlist` | array of strings | no | Force people to use only corporate emails for sign-up. Default is `null`, meaning there is no restriction. | | `dsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded DSA key. Default is `0` (no restriction). `-1` disables DSA keys. | | `ecdsa_key_restriction` | integer | no | The minimum allowed curve size (in bits) of an uploaded ECDSA key. Default is `0` (no restriction). `-1` disables ECDSA keys. | @@ -351,9 +356,9 @@ listed in the descriptions of the relevant settings. | `html_emails_enabled` | boolean | no | Enable HTML emails. | | `import_sources` | array of strings | no | Sources to allow project import from, possible values: `github`, `bitbucket`, `bitbucket_server`, `gitlab`, `fogbugz`, `git`, `gitlab_project`, `gitea`, `manifest`, and `phabricator`. | | `in_product_marketing_emails_enabled` | boolean | no | Enable [in-product marketing emails](../user/profile/notifications.md#global-notification-settings). Enabled by default. | -| `inactive_projects_delete_after_months` | integer | no | If `delete_inactive_projects` is `true`, the time (in months) to wait before deleting inactive projects. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. | -| `inactive_projects_min_size_mb` | integer | no | If `delete_inactive_projects` is `true`, the minimum repository size for projects to be checked for inactivity. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. | -| `inactive_projects_send_warning_email_after_months` | integer | no | If `delete_inactive_projects` is `true`, sets the time (in months) to wait before emailing maintainers that the project will be deleted because it is inactive. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. | +| `inactive_projects_delete_after_months` | integer | no | If `delete_inactive_projects` is `true`, the time (in months) to wait before deleting inactive projects. Default is `2`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. | +| `inactive_projects_min_size_mb` | integer | no | If `delete_inactive_projects` is `true`, the minimum repository size for projects to be checked for inactivity. Default is `0`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. | +| `inactive_projects_send_warning_email_after_months` | integer | no | If `delete_inactive_projects` is `true`, sets the time (in months) to wait before emailing maintainers that the project is scheduled be deleted because it is inactive. Default is `1`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. | | `invisible_captcha_enabled` | boolean | no | Enable Invisible CAPTCHA spam detection during sign-up. Disabled by default. | | `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Disabled by default.| | `keep_latest_artifact` | boolean | no | Prevent the deletion of the artifacts from the most recent successful jobs, regardless of the expiry time. Enabled by default. | @@ -364,9 +369,10 @@ listed in the descriptions of the relevant settings. | `maintenance_mode` **(PREMIUM)** | boolean | no | When instance is in maintenance mode, non-administrative users can sign in with read-only access and make read-only API requests. | | `max_artifacts_size` | integer | no | Maximum artifacts size in MB. | | `max_attachment_size` | integer | no | Limit attachment size in MB. | +| `max_export_size` | integer | no | Maximum export size in MB. 0 for unlimited. Default = 0 (unlimited). | | `max_import_size` | integer | no | Maximum import size in MB. 0 for unlimited. Default = 0 (unlimited) [Modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. | | `max_pages_size` | integer | no | Maximum size of pages repositories in MB. | -| `max_personal_access_token_lifetime` **(ULTIMATE SELF)** | integer | no | Maximum allowable lifetime for personal access tokens in days. | +| `max_personal_access_token_lifetime` **(ULTIMATE SELF)** | integer | no | Maximum allowable lifetime for access tokens in days. | | `max_ssh_key_lifetime` **(ULTIMATE SELF)** | integer | no | Maximum allowable lifetime for SSH keys in days. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.6. | | `metrics_method_call_threshold` | integer | no | A method call is only tracked when it takes longer than the given amount of milliseconds. | | `mirror_available` | boolean | no | Allow repository mirroring to configured by project Maintainers. If disabled, only Administrators can configure repository mirroring. | @@ -383,13 +389,13 @@ listed in the descriptions of the relevant settings. | `performance_bar_allowed_group_path` | string | no | Path of the group that is allowed to toggle the performance bar. | | `performance_bar_enabled` | boolean | no | (Deprecated: Pass `performance_bar_allowed_group_path: nil` instead) Allow enabling the performance bar. | | `personal_access_token_prefix` | string | no | Prefix for all generated personal access tokens. | +| `pipeline_limit_per_project_user_sha` | integer | no | Maximum number of pipeline creation requests per minute per user and commit. Disabled by default. | | `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable PlantUML integration. Default is `false`. | | `plantuml_url` | string | required by: `plantuml_enabled` | The PlantUML instance URL for integration. | | `polling_interval_multiplier` | decimal | no | Interval multiplier used by endpoints that perform polling. Set to `0` to disable polling. | | `project_export_enabled` | boolean | no | Enable project export. | | `prometheus_metrics_enabled` | boolean | no | Enable Prometheus metrics. | | `protected_ci_variables` | boolean | no | CI/CD variables are protected by default. | -| `pseudonymizer_enabled` **(PREMIUM)** | boolean | no | When enabled, GitLab runs a background job that produces pseudonymized CSVs of the GitLab database to upload to your configured object storage directory. | `push_event_activities_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether individual push events or bulk push events are created. [Bulk push events are created](../user/admin_area/settings/push_event_activities_limit.md) if it surpasses that value. | | `push_event_hooks_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether webhooks and services fire or not. Webhooks and services aren't submitted if it surpasses that value. | | `rate_limiting_response_text` | string | no | When rate limiting is enabled via the `throttle_*` settings, send this plain text response when a rate limit is exceeded. 'Retry later' is sent if this is blank. | @@ -423,6 +429,7 @@ listed in the descriptions of the relevant settings. | `slack_app_enabled` **(PREMIUM)** | boolean | no | (**If enabled, requires:** `slack_app_id`, `slack_app_secret` and `slack_app_secret`) Enable Slack app. | | `slack_app_id` **(PREMIUM)** | string | required by: `slack_app_enabled` | The app ID of the Slack-app. | | `slack_app_secret` **(PREMIUM)** | string | required by: `slack_app_enabled` | The app secret of the Slack-app. | +| `slack_app_signing_secret` **(PREMIUM)** | string | no | The signing secret of the Slack-app. | | `slack_app_verification_token` **(PREMIUM)** | string | required by: `slack_app_enabled` | The verification token of the Slack-app. | | `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50MB).| | `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | diff --git a/doc/api/status_checks.md b/doc/api/status_checks.md index 3671ddbd138..92e003bf80d 100644 --- a/doc/api/status_checks.md +++ b/doc/api/status_checks.md @@ -31,7 +31,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/status_checks "id": 2, "name": "Rule 1", "external_url": "https://gitlab.com/test-endpoint", - "status": "pass" + "status": "passed" }, { "id": 1, @@ -47,11 +47,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/status_checks > - Introduced in GitLab 14.9, `passed` status to pass external status checks. Introduced [with a flag](../administration/feature_flags.md) named `status_checks_add_status_field`. Disabled by default. > - Introduced in GitLab 14.9, `failed` status to fail external status checks. Introduced [with a flag](../administration/feature_flags.md) named `status_checks_add_status_field`. Disabled by default. > - `pass` status to pass checks is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/339039) in GitLab 14.9. Replaced with `passed`. - -FLAG: -On self-managed GitLab, by default setting `passed` instead of `pass` is unavailable. Also, setting `failed` is unavailable by default. To support -setting `passed` and `failed` instead of only `pass`, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named -`status_checks_add_status_field`. On GitLab.com, this feature is not available. +> - Support for `failed` and `passed` [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/353836) in GitLab 15.0 and feature flag removed. For a single merge request, use the API to inform GitLab that a merge request has passed a check by an external service. To set the status of an external check, the personal access token used must belong to a user with at least the Developer role on the target project of the merge request. @@ -64,14 +60,13 @@ POST /projects/:id/merge_requests/:merge_request_iid/status_check_responses **Parameters:** -| Attribute | Type | Required | Description | -| -------------------------- | ------- | -------- | ---------------------------------------------------------------------------- | -| `id` | integer | yes | ID of a project | -| `merge_request_iid` | integer | yes | IID of a merge request | -| `sha` | string | yes | SHA at `HEAD` of the source branch | -| `external_status_check_id` | integer | yes | ID of an external status check | -| `status` | string | no | Set to `passed` to pass the check or `failed` to fail it (GitLab 14.9 and later with feature flag `status_checks_add_status_field` enabled) | -| `status` | string | no | Set to `pass` to pass the check (GitLab 14.0 to GitLab 14.8, and GitLab 14.9 and later with feature flag `status_checks_add_status_field` disabled) | +| Attribute | Type | Required | Description | +| -------------------------- | ------- | -------- |----------------------------------------------------------| +| `id` | integer | yes | ID of a project | +| `merge_request_iid` | integer | yes | IID of a merge request | +| `sha` | string | yes | SHA at `HEAD` of the source branch | +| `external_status_check_id` | integer | yes | ID of an external status check | +| `status` | string | no | Set to `passed` to pass the check or `failed` to fail it | NOTE: `sha` must be the SHA at the `HEAD` of the merge request's source branch. diff --git a/doc/api/topics.md b/doc/api/topics.md index 1246e36f2cb..78f32fb8fa0 100644 --- a/doc/api/topics.md +++ b/doc/api/topics.md @@ -38,21 +38,24 @@ Example response: [ { "id": 1, - "name": "GitLab", + "name": "gitlab", + "title": "GitLab", "description": "GitLab is an open source end-to-end software development platform with built-in version control, issue tracking, code review, CI/CD, and more.", "total_projects_count": 1000, "avatar_url": "http://www.gravatar.com/avatar/a0d477b3ea21970ce6ffcbb817b0b435?s=80&d=identicon" }, { "id": 3, - "name": "Git", + "name": "git", + "title": "Git", "description": "Git is a free and open source distributed version control system designed to handle everything from small to very large projects with speed and efficiency.", "total_projects_count": 900, "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon" }, { "id": 2, - "name": "Git LFS", + "name": "git-lfs", + "title": "Git LFS", "description": null, "total_projects_count": 300, "avatar_url": null @@ -85,7 +88,8 @@ Example response: ```json { "id": 1, - "name": "GitLab", + "name": "gitlab", + "title": "GitLab", "description": "GitLab is an open source end-to-end software development platform with built-in version control, issue tracking, code review, CI/CD, and more.", "total_projects_count": 1000, "avatar_url": "http://www.gravatar.com/avatar/a0d477b3ea21970ce6ffcbb817b0b435?s=80&d=identicon" @@ -112,7 +116,8 @@ Supported attributes: | Attribute | Type | Required | Description | | ------------- | ------- | ---------------------- | ----------- | -| `name` | string | **{check-circle}** Yes | Name | +| `name` | string | **{check-circle}** Yes | Slug (name) | +| `title` | string | **{check-circle}** Yes | Title | | `avatar` | file | **{dotted-circle}** No | Avatar | | `description` | string | **{dotted-circle}** No | Description | @@ -120,7 +125,7 @@ Example request: ```shell curl --request POST \ - --data "name=topic1" \ + --data "name=topic1&title=Topic 1" \ --header "PRIVATE-TOKEN: <your_access_token>" \ "https://gitlab.example.com/api/v4/topics" ``` @@ -131,6 +136,7 @@ Example response: { "id": 1, "name": "topic1", + "title": "Topic 1", "description": null, "total_projects_count": 0, "avatar_url": null @@ -152,7 +158,8 @@ Supported attributes: | `id` | integer | **{check-circle}** Yes | ID of project topic | | `avatar` | file | **{dotted-circle}** No | Avatar | | `description` | string | **{dotted-circle}** No | Description | -| `name` | string | **{dotted-circle}** No | Name | +| `name` | string | **{dotted-circle}** No | Slug (name) | +| `title` | string | **{dotted-circle}** No | Title | Example request: @@ -169,6 +176,7 @@ Example response: { "id": 1, "name": "topic1", + "title": "Topic 1", "description": null, "total_projects_count": 0, "avatar_url": null diff --git a/doc/api/users.md b/doc/api/users.md index 7b4962735e6..5e41a0f6258 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -105,7 +105,7 @@ parameter `without_project_bots=true`. GET /users?without_project_bots=true ``` -### For admins +### For administrators > The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. @@ -309,11 +309,12 @@ Parameters: "work_information": null, "followers": 1, "following": 1, - "local_time": "3:38 PM" + "local_time": "3:38 PM", + "is_followed": false } ``` -### For admin +### For administrator > The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. @@ -452,7 +453,7 @@ Parameters: | Attribute | Required | Description | | :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `admin` | No | User is admin - true or false (default) | +| `admin` | No | User is an administrator - true or false (default) | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | @@ -466,7 +467,7 @@ Parameters: | `linkedin` | No | LinkedIn | | `location` | No | User's location | | `name` | Yes | Name | -| `note` | No | Admin notes for this user | +| `note` | No | Administrator notes for this user | | `organization` | No | Organization name | | `password` | No | Password | | `private_profile` | No | User's profile is private - true, false (default), or null (is converted to false) | @@ -496,7 +497,7 @@ Parameters: | Attribute | Required | Description | | :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `admin` | No | User is admin - true or false (default) | +| `admin` | No | User is an administrator - true or false (default) | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | @@ -510,7 +511,7 @@ Parameters: | `linkedin` | No | LinkedIn | | `location` | No | User's location | | `name` | No | Name | -| `note` | No | Admin notes for this user | +| `note` | No | Administration notes for this user | | `organization` | No | Organization name | | `password` | No | Password | | `private_profile` | No | User's profile is private - true, false (default), or null (is converted to false) | @@ -618,7 +619,7 @@ GET /user Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit` parameters. -## List current user (for admins) +## List current user (for administrators) > The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. @@ -1894,7 +1895,7 @@ Example response: } ``` -## Get user activities (admin only) +## Get user activities (administrator only) NOTE: This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above. @@ -1950,7 +1951,7 @@ Example response: `last_activity_at` is deprecated. Use `last_activity_on` instead. -## User memberships (admin only) +## User memberships (administrator only) > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20532) in GitLab 12.8. diff --git a/doc/api/wikis.md b/doc/api/wikis.md index b7c9d0d6008..0122872becf 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -68,7 +68,7 @@ GET /projects/:id/wikis/:slug | Attribute | Type | Required | Description | | --------- | ------- | -------- | --------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) | -| `slug` | string | yes | URLencoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | +| `slug` | string | yes | URL encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | | `render_html` | boolean | no | Return the rendered HTML of the wiki page | | `version` | string | no | Wiki page version sha | diff --git a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md index 52e9b48419c..53ea56b8724 100644 --- a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md +++ b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md @@ -105,7 +105,7 @@ essentially makes it available on all levels: - projects Various traversal queries are already available on `Namespaces` to query the -group hierarchy. `Projects` represents the leaf nodes in the hierarchy, but with +group hierarchy. `Projects` represent the leaf nodes in the hierarchy, but with the introduction of `ProjectNamespace`, these traversal queries can be used to retrieve projects as well. @@ -132,7 +132,7 @@ epic. The initial iteration will provide a framework to house features under `Namespaces`. Stage groups will eventually need to migrate their own features and functionality over to `Namespaces`. This may impact these features in unexpected ways. Therefore, to minimize UX debt and maintain product consistency, stage groups will have to consider a number of factors when migrating their features over to `Namespaces`: 1. **Conceptual model**: What are the current and future state conceptual models of these features ([see object modeling for designers](https://hpadkisson.medium.com/object-modeling-for-designers-an-introduction-7871bdcf8baf))? These should be documented in Pajamas (example: [merge requests](https://design.gitlab.com/objects/merge-request)). -1. **Merge conflicts**: What inconsistencies are there across project, group, and admin levels? How might these be addressed? For an example of how we rationalized this for labels, please see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338820). +1. **Merge conflicts**: What inconsistencies are there across project, group, and administrator levels? How might these be addressed? For an example of how we rationalized this for labels, please see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338820). 1. **Inheritance & information flow**: How is information inherited across our container hierarchy currently? How might this be impacted if complying with the new [inheritance behavior](https://gitlab.com/gitlab-org/gitlab/-/issues/343316) framework? 1. **Settings**: Where can settings for this feature be found currently? How will these be impacted by `Namespaces`? 1. **Access**: Who can access this feature and is that impacted by the new container structure? Are there any role or privacy considerations? diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 963977be549..5105f2e08e2 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -339,7 +339,7 @@ A more detailed list of all tasks, as well as periodic progress updates can be f ## Relevant Links -- [Allow admin to run garbage collection with zero downtime](https://gitlab.com/groups/gitlab-org/-/epics/2313) +- [Allow administrators to run garbage collection with zero downtime](https://gitlab.com/groups/gitlab-org/-/epics/2313) - [Proposal for continuous, on-demand online garbage collection](https://gitlab.com/gitlab-org/container-registry/-/issues/199) - [Gradual migration proposal for the GitLab.com container registry](https://gitlab.com/gitlab-org/container-registry/-/issues/191) - [Create a self-serve registry deployment](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/316) diff --git a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md index 9309c581d54..b4614cde9d4 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md +++ b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md @@ -133,8 +133,8 @@ You can find more information on table partitioning for PostgreSQL in the [documentation page for table partitioning](https://www.postgresql.org/docs/12/ddl-partitioning.html). Partitioning by date intervals (for example, month, year) allows us to create much smaller tables -(partitions) for each date interval and only access the most recent partition(s) for any -application related operation. +(partitions) for each date interval and only access the most recent partitions for any +application-related operation. We have to set the partitioning key based on the date interval of interest, which may depend on two factors: @@ -214,7 +214,7 @@ offloading metadata but only for the case of old data. In the simplest use case we can provide fast and direct access to recent data, while allowing users to download an archive with older data. This is an option evaluated in the `audit_events` use case. Depending on the country and industry, audit events may have a very long retention period, while -only the past month(s) of data are actively accessed through GitLab interface. +only the past months of data are actively accessed through GitLab interface. Additional use cases may include exporting data to a data warehouse or other types of data stores as they may be better suited for processing that type of data. An example can be JSON logs that we diff --git a/doc/architecture/blueprints/database_scaling/size-limits.md b/doc/architecture/blueprints/database_scaling/size-limits.md index d63aa3bd4e8..a0508488620 100644 --- a/doc/architecture/blueprints/database_scaling/size-limits.md +++ b/doc/architecture/blueprints/database_scaling/size-limits.md @@ -126,7 +126,7 @@ In order to maintain and improve operational stability and lessen development bu This target is *pragmatic*: We understand table sizes depend on feature usage, code changes and other factors - which all change over time. We may not always find solutions where we can tightly limit the size of physical tables once and for all. That is acceptable though and we primarily aim to keep the situation on GitLab.com under control. We adapt our efforts to the situation present on GitLab.com and will re-evaluate frequently. -While there are changes we can make that lead to a constant maximum physical table size over time, this doesn't need to be the case necessarily. Consider for example hash partitioniong, which breaks a table down into a static number of partitions. With data growth over time, individual partitions will also grow in size and may eventually reach the threshold size again. We strive to get constant table sizes, but it is acceptable to ship easier solutions that don't have this characteristic but improve the situation for a considerable amount of time. +While there are changes we can make that lead to a constant maximum physical table size over time, this doesn't need to be the case necessarily. Consider for example hash partitioning, which breaks a table down into a static number of partitions. With data growth over time, individual partitions will also grow in size and may eventually reach the threshold size again. We strive to get constant table sizes, but it is acceptable to ship easier solutions that don't have this characteristic but improve the situation for a considerable amount of time. As such, the target size of a physical table after refactoring depends on the situation and there is no hard rule for it. We suggest to consider historic data growth and forecast when physical tables will reach the threshold of 100 GB again. This allows us to understand how long a particular solution is expected to last until the model has to be revisited. diff --git a/doc/architecture/blueprints/database_testing/index.md b/doc/architecture/blueprints/database_testing/index.md index 8c0cb550d61..22857abf176 100644 --- a/doc/architecture/blueprints/database_testing/index.md +++ b/doc/architecture/blueprints/database_testing/index.md @@ -100,7 +100,7 @@ The short-term goal is detailed in [this epic](https://gitlab.com/groups/gitlab- ### Mid-term - Improved feedback, query testing and background migration testing -Mid-term, we plan to expand the level of detail the testing pipeline reports back to the merge requet and expand its scope to cover query testing, too. By doing so, we use our experience from database code reviews and using thin-clone technology and bring this back closer to the GitLab workflow. Instead of reaching out to different tools (`postgres.ai`, `joe`, Slack, plan visualizations, and so on) we bring this back to GitLab and working directly on the merge request. +Mid-term, we plan to expand the level of detail the testing pipeline reports back to the merge request and expand its scope to cover query testing, too. By doing so, we use our experience from database code reviews and using thin-clone technology and bring this back closer to the GitLab workflow. Instead of reaching out to different tools (`postgres.ai`, `joe`, Slack, plan visualizations, and so on) we bring this back to GitLab and working directly on the merge request. Secondly, we plan to cover background migrations testing, too. These are typically data migrations that are scheduled to run over a long period of time. The success of both the scheduling phase and the job execution phase typically depends a lot on data distribution - which only surfaces when running these migrations on actual production data. In order to become confident about a background migration, we plan to provide the following feedback: diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 777bbf6053f..88d59b1f223 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -2,12 +2,11 @@ stage: Verify group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: index, concepts, howto --- # Caching in GitLab CI/CD **(FREE)** -A cache is one or more files that a job downloads and saves. Subsequent jobs that use +A cache is one or more files a job downloads and saves. Subsequent jobs that use the same cache don't have to download the files again, so they execute more quickly. To learn how to define the cache in your `.gitlab-ci.yml` file, @@ -31,7 +30,7 @@ can't link to files outside it. - Subsequent pipelines can use the cache. - Subsequent jobs in the same pipeline can use the cache, if the dependencies are identical. - Different projects cannot share the cache. -- Protected and non-protected branches do not share the cache. +- By default, protected and non-protected branches [do not share the cache](#cache-key-names). However, you can [change this behavior](#use-the-same-cache-for-all-branches). ### Artifacts @@ -447,13 +446,11 @@ is stored on the machine where GitLab Runner is installed. The location also dep If you use cache and artifacts to store the same path in your jobs, the cache might be overwritten because caches are restored before artifacts. -### Segregation of caches between protected and non-protected branches +#### Cache key names > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330047) in GitLab 15.0. A suffix is added to the cache key, with the exception of the [fallback cache key](#use-a-fallback-cache-key). -This is done in order to prevent cache poisoning that might occur through manipulation of the cache in a non-protected -branch. Any subsequent protected-branch jobs would then potentially use a poisoned cache from the preceding job. As an example, assuming that `cache.key` is set to `$CI_COMMIT_REF_SLUG`, and that we have two branches `main` and `feature`, then the following table represents the resulting cache keys: @@ -463,6 +460,24 @@ and `feature`, then the following table represents the resulting cache keys: | `main` | `main-protected` | | `feature` | `feature-non_protected` | +##### Use the same cache for all branches + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361643) in GitLab 15.0. + +If you do not want to use [cache key names](#cache-key-names), +you can have all branches (protected and unprotected) use the same cache. + +The cache separation with [cache key names](#cache-key-names) is a security feature +and should only be disabled in an environment where all users with Developer role are highly trusted. + +To use the same cache for all branches: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **General pipelines**. +1. Clear the **Use separate caches for protected branches** checkbox. +1. Select **Save changes**. + ### How archiving and extracting works This example shows two jobs in two consecutive stages: @@ -552,12 +567,10 @@ The next time the pipeline runs, the cache is stored in a different location. ### Clear the cache manually -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41249) in GitLab 10.4. - You can clear the cache in the GitLab UI: 1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **CI/CD > Pipelines** page. +1. On the left sidebar, select **CI/CD > Pipelines**. 1. In the top right, select **Clear runner caches**. On the next commit, your CI/CD jobs use a new cache. @@ -576,7 +589,7 @@ If you have a cache mismatch, follow these steps to troubleshoot. | You use multiple standalone runners (not in autoscale mode) attached to one project without a shared cache. | Use only one runner for your project or use multiple runners with distributed cache enabled. | | You use runners in autoscale mode without a distributed cache enabled. | Configure the autoscale runner to use a distributed cache. | | The machine the runner is installed on is low on disk space or, if you've set up distributed cache, the S3 bucket where the cache is stored doesn't have enough space. | Make sure you clear some space to allow new caches to be stored. There's no automatic way to do this. | -| You use the same `key` for jobs where they cache different paths. | Use different cache keys to that the cache archive is stored to a different location and doesn't overwrite wrong caches. | +| You use the same `key` for jobs where they cache different paths. | Use different cache keys so that the cache archive is stored to a different location and doesn't overwrite wrong caches. | #### Cache mismatch example 1 diff --git a/doc/ci/chatops/index.md b/doc/ci/chatops/index.md index c785e2f29ea..21b970536bc 100644 --- a/doc/ci/chatops/index.md +++ b/doc/ci/chatops/index.md @@ -73,7 +73,7 @@ stages: hello-world: stage: chatops rules: - - if: '$CI_PIPELINE_SOURCE == "chat"' + - if: $CI_PIPELINE_SOURCE == "chat" script: - echo "Hello World" ``` @@ -93,7 +93,7 @@ stages: ls: stage: chatops rules: - - if: '$CI_PIPELINE_SOURCE == "chat"' + - if: $CI_PIPELINE_SOURCE == "chat" script: - echo "This command will not be shown." - echo -e "section_start:$( date +%s ):chat_reply\r\033[0K\n$( ls -la )\nsection_end:$( date +%s ):chat_reply\r\033[0K" diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index 80e4d8fbe27..c89ce2675e4 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -59,7 +59,7 @@ Some credentials are required to be able to run `aws` commands: | `AWS_DEFAULT_REGION` | Your region code | NOTE: - When you create a variable it's set to be [protected by default](../variables/index.md#protect-a-cicd-variable). If you want to use the `aws` commands on branches or tags that are not protected, make sure to uncheck the **Protect variable** checkbox. + When you create a variable it's set to be [protected by default](../variables/index.md#protected-cicd-variables). If you want to use the `aws` commands on branches or tags that are not protected, make sure to uncheck the **Protect variable** checkbox. 1. You can now use `aws` commands in the `.gitlab-ci.yml` file of this project: diff --git a/doc/ci/cloud_services/aws/index.md b/doc/ci/cloud_services/aws/index.md index aa38562c866..dc8dfd95899 100644 --- a/doc/ci/cloud_services/aws/index.md +++ b/doc/ci/cloud_services/aws/index.md @@ -90,3 +90,4 @@ This error can occur for multiple reasons: - The cloud administrator has not configured the project to use OIDC with GitLab. - The role is restricted from being run on the branch or tag. See [configure a conditional role](../index.md). +- `StringEquals` is used instead of `StringLike` when using a wildcard condition. See [related issue](https://gitlab.com/guided-explorations/aws/configure-openid-connect-in-aws/-/issues/2#note_852901934). diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index 65c907b8e7b..2e514fd0f0a 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -31,7 +31,7 @@ If you disable GitLab CI/CD in a project: - Existing jobs and pipelines are not deleted. Re-enable CI/CD to access them again. The project or instance settings do not enable or disable pipelines run in an -[external integration](../user/project/integrations/overview.md#integrations-listing). +[external integration](../user/project/integrations/index.md#available-integrations). ## Enable CI/CD in a project diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md index 2a5927a96c2..23fa5d45196 100644 --- a/doc/ci/environments/deployment_approvals.md +++ b/doc/ci/environments/deployment_approvals.md @@ -34,7 +34,7 @@ To configure deployment approvals for a project: ### Create a deployment job -Create a deployment job in the `.gitlab-ci.yaml` file of the desired project. The job does **not** need to be manual (`when: manual`). +Create a deployment job in the `.gitlab-ci.yml` file of the desired project. The job does **not** need to be manual (`when: manual`). Example: @@ -92,7 +92,8 @@ Maintainer role. #### Multiple approval rules -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) in GitLab 14.10 with a flag named `deployment_approval_rules`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) in GitLab 14.10 with a flag named `deployment_approval_rules`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) in GitLab 15.0. [Feature flag `deployment_approval_rules`](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) removed. 1. Using the [REST API](../../api/group_protected_environments.md#protect-an-environment). 1. `deploy_access_levels` represents which entity can execute the deployment job. @@ -138,7 +139,9 @@ To approve or reject a deployment to a protected environment using the UI: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Deployments > Environments**. +1. Select the environment's name. 1. In the deployment's row, select **Approval options** (**{thumb-up}**). +1. Optional. Add a comment which describes your reason for approving or rejecting the deployment. 1. Select **Approve** or **Reject**. NOTE: diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index 0e73dc4f7cd..5b2e2045bdc 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Deployment safety **(FREE)** -Deployment jobs can be more sensitive than other jobs in a pipeline, +[Deployment jobs](../jobs/#deployment-jobs) are a specific kind of CI/CD +job. They can be more sensitive than other jobs in a pipeline, and might need to be treated with extra care. GitLab has several features that help maintain deployment security and stability. @@ -64,14 +65,14 @@ For more information, see [Resource Group documentation](../resource_groups/inde ## Skip outdated deployment jobs -The execution order of pipeline jobs can vary from run to run, which could cause -undesired behavior. For example, a deployment job in a newer pipeline could -finish before a deployment job in an older pipeline. -This creates a race condition where the older deployment finished later, +The effective execution order of pipeline jobs can vary from run to run, which +could cause undesired behavior. For example, a [deployment job](../jobs/#deployment-jobs) +in a newer pipeline could finish before a deployment job in an older pipeline. +This creates a race condition where the older deployment finishes later, overwriting the "newer" deployment. You can ensure that older deployment jobs are cancelled automatically when a newer deployment -runs by enabling the [Skip outdated deployment jobs](../pipelines/settings.md#skip-outdated-deployment-jobs) feature. +job is started by enabling the [Skip outdated deployment jobs](../pipelines/settings.md#skip-outdated-deployment-jobs) feature. Example of a problematic pipeline flow **before** enabling Skip outdated deployment jobs: @@ -85,7 +86,7 @@ The improved pipeline flow **after** enabling Skip outdated deployment jobs: 1. Pipeline-A is created on the default branch. 1. Later, Pipeline-B is created on the default branch (with a newer SHA). 1. The `deploy` job in Pipeline-B finishes first, and deploys the newer code. -1. The `deploy` job in Pipeline-A is automatically cancelled, so that it doesn't overwrite the deployment from the newer pipeline. +1. The `deploy` job in Pipeline-A was automatically cancelled, so that it doesn't overwrite the deployment from the newer pipeline. ## Prevent deployments during deploy freeze windows @@ -111,7 +112,7 @@ for an explanation of these roles and the permissions of each. Production secrets are needed to deploy successfully. For example, when deploying to the cloud, cloud providers require these secrets to connect to their services. In the project settings, you can -define and protect CI/CD variables for these secrets. [Protected variables](../variables/index.md#protect-a-cicd-variable) +define and protect CI/CD variables for these secrets. [Protected variables](../variables/index.md#protected-cicd-variables) are only passed to pipelines running on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). The other pipelines don't get the protected variable. You can also @@ -132,7 +133,7 @@ permission model that isolates the CD permissions from the original project and original users with the Maintainer role for the project from accessing the production secret and CD configuration. You can connect the CD project to your development projects by using [multi-project pipelines](../pipelines/multi_project_pipelines.md). -## Protect `gitlab-ci.yml` from change +## Protect `.gitlab-ci.yml` from change A `.gitlab-ci.yml` may contain rules to deploy an application to the production server. This deployment usually runs automatically after pushing a merge request. To prevent developers from diff --git a/doc/ci/environments/incremental_rollouts.md b/doc/ci/environments/incremental_rollouts.md index bc52fd1f16c..d359ed078ff 100644 --- a/doc/ci/environments/incremental_rollouts.md +++ b/doc/ci/environments/incremental_rollouts.md @@ -114,7 +114,7 @@ available, [demonstrating configuration of timed rollouts](https://gitlab.com/gl ## Blue-Green Deployment NOTE: -As of GitLab 13.7, teams can leverage an Ingress annotation and [set traffic weight](../../user/project/canary_deployments.md#how-to-change-the-traffic-weight-on-a-canary-ingress) +As of GitLab 13.7, teams can leverage an Ingress annotation and [set traffic weight](../../user/project/canary_deployments.md#how-to-change-the-traffic-weight-on-a-canary-ingress-deprecated) as an alternative approach to the blue-green deployment strategy documented here. Also sometimes known as A/B deployment or red-black deployment, this technique is used to reduce diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 3822b32181f..7e1ef5efaa5 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -560,11 +560,8 @@ GitLab automatically triggers the `stop_review_app` job to stop the environment. #### Multiple stop actions for an environment -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22456) in GitLab 14.10 [with a flag](../../administration/feature_flags.md) named `environment_multiple_stop_actions`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `environment_multiple_stop_actions`. -On GitLab.com, this feature is not available. We are enabling in phases and the status can be tracked in [issue 358911](https://gitlab.com/gitlab-org/gitlab/-/issues/358911). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22456) in GitLab 14.10 [with a flag](../../administration/feature_flags.md) named `environment_multiple_stop_actions`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/358911) in GitLab 15.0. [Feature flag `environment_multiple_stop_actions`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86685) removed. This feature is useful when you need to perform multiple **parallel** stop actions on an environment. @@ -645,17 +642,14 @@ To delete a stopped environment in the GitLab UI: 1. Next to the environment you want to delete, select **Delete environment**. 1. On the confirmation dialog box, select **Delete environment**. -### Prepare an environment without creating a deployment +### Access an environment for preparation or verification purposes > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208655) in GitLab 13.2. -By default, when GitLab CI/CD runs a job for a specific environment, it -triggers a deployment and [(optionally) cancels outdated -deployments](deployment_safety.md#ensure-only-one-deployment-job-runs-at-a-time). +You can define a job that accesses an environment for various purposes, such as verification or preparation. This +effectively bypasses deployment creation, so that you can adjust your CD workflow more accurately. -To use an environment without creating a new deployment, and without -cancelling outdated deployments, append the keyword `action: prepare` to your -job: +To do so, add either `action: prepare`, `action: verify`, or `action: access` to the `environment` section of your job: ```yaml build: @@ -668,8 +662,8 @@ build: url: https://staging.example.com ``` -This gives you access to [environment-scoped variables](#scope-environments-with-specs), -and can be used to [protect builds from unauthorized access](protected_environments.md). +This gives you access to environment-scoped variables, and can be used to protect builds from unauthorized access. Also, +it's effective to avoid the [skip outdated deployment jobs](deployment_safety.md#skip-outdated-deployment-jobs) feature. ### Group similar environments diff --git a/doc/ci/examples/semantic-release.md b/doc/ci/examples/semantic-release.md index c74af852a9a..2e9e2dd33bf 100644 --- a/doc/ci/examples/semantic-release.md +++ b/doc/ci/examples/semantic-release.md @@ -35,7 +35,7 @@ You can also view or fork the complete [example source](https://gitlab.com/gitla } ``` -1. Update the `files` key with glob pattern(s) that selects all files that should be included in the published module. More information about `files` can be found [in npm's documentation](https://docs.npmjs.com/cli/v6/configuring-npm/package-json/#files). +1. Update the `files` key with glob patterns that selects all files that should be included in the published module. More information about `files` can be found [in npm's documentation](https://docs.npmjs.com/cli/v6/configuring-npm/package-json/#files). 1. Add a `.gitignore` file to the project to avoid committing `node_modules`: diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 23055514839..5f22fb894e5 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -83,4 +83,4 @@ You can check documentation for your specific OS to learn how to find and displa hidden files. If there is no `.gitmodules` file, it's possible the submodule settings are in a -[gitconfig](https://www.atlassian.com/git/tutorials/setting-up-a-repository/git-config) file. +[`git config`](https://www.atlassian.com/git/tutorials/setting-up-a-repository/git-config) file. diff --git a/doc/ci/jobs/index.md b/doc/ci/jobs/index.md index e589fd8b045..42827dc2d48 100644 --- a/doc/ci/jobs/index.md +++ b/doc/ci/jobs/index.md @@ -392,5 +392,5 @@ deploy me: The behavior of deployment jobs can be controlled with [deployment safety](../environments/deployment_safety.md) settings like -[skipping outdated deployment jobs](../environments/deployment_safety.md#prevent-deployments-during-deploy-freeze-windows) +[skipping outdated deployment jobs](../environments/deployment_safety.md#skip-outdated-deployment-jobs) and [ensuring only one deployment job runs at a time](../environments/deployment_safety.md#ensure-only-one-deployment-job-runs-at-a-time). diff --git a/doc/ci/jobs/job_control.md b/doc/ci/jobs/job_control.md index 3a302cf352b..83747f36597 100644 --- a/doc/ci/jobs/job_control.md +++ b/doc/ci/jobs/job_control.md @@ -40,10 +40,10 @@ The following example uses `if` to define that the job runs in only two specific job: script: echo "Hello, Rules!" rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual allow_failure: true - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: $CI_PIPELINE_SOURCE == "schedule" ``` - If the pipeline is for a merge request, the first rule matches, and the job @@ -67,9 +67,9 @@ run them in all other cases: job: script: echo "Hello, Rules!" rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: never - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: $CI_PIPELINE_SOURCE == "schedule" when: never - when: on_success ``` @@ -118,7 +118,7 @@ For example: docker build: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - - if: '$VAR == "string value"' + - if: $VAR == "string value" changes: # Include the job and set to when:manual if any of the follow paths match a modified file. - Dockerfile - docker/scripts/* @@ -160,7 +160,7 @@ For example: job: script: echo "This job creates double pipelines!" rules: - - if: '$CUSTOM_VARIABLE == "false"' + - if: $CUSTOM_VARIABLE == "false" when: never - when: always ``` @@ -181,7 +181,7 @@ To avoid duplicate pipelines, you can: job: script: echo "This job does NOT create double pipelines!" rules: - - if: '$CUSTOM_VARIABLE == "true" && $CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CUSTOM_VARIABLE == "true" && $CI_PIPELINE_SOURCE == "merge_request_event" ``` You can also avoid duplicate pipelines by changing the job rules to avoid either push (branch) @@ -195,7 +195,7 @@ without `workflow: rules`: job: script: echo "This job does NOT create double pipelines!" rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_PIPELINE_SOURCE == "push" when: never - when: always ``` @@ -207,8 +207,8 @@ You should not include both push and merge request pipelines in the same job wit job: script: echo "This job creates double pipelines!" rules: - - if: '$CI_PIPELINE_SOURCE == "push"' - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" ``` Also, do not mix `only/except` jobs with `rules` jobs in the same pipeline. @@ -222,7 +222,7 @@ job-with-no-rules: job-with-rules: script: echo "This job runs in merge request pipelines." rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" ``` For every change pushed to the branch, duplicate pipelines run. One @@ -259,10 +259,10 @@ add the job to any other pipeline type. job: script: echo "Hello, Rules!" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: $CI_PIPELINE_SOURCE == "schedule" when: manual allow_failure: true - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_PIPELINE_SOURCE == "push" ``` The following example runs the job as a `when: on_success` job in [merge request pipelines](../pipelines/merge_request_pipelines.md) @@ -272,25 +272,25 @@ and scheduled pipelines. It does not run in any other pipeline type. job: script: echo "Hello, Rules!" rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "schedule" ``` Other commonly used variables for `if` clauses: - `if: $CI_COMMIT_TAG`: If changes are pushed for a tag. - `if: $CI_COMMIT_BRANCH`: If changes are pushed to any branch. -- `if: '$CI_COMMIT_BRANCH == "main"'`: If changes are pushed to `main`. -- `if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'`: If changes are pushed to the default +- `if: $CI_COMMIT_BRANCH == "main"`: If changes are pushed to `main`. +- `if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH`: If changes are pushed to the default branch. Use when you want to have the same configuration in multiple projects with different default branches. -- `if: '$CI_COMMIT_BRANCH =~ /regex-expression/'`: If the commit branch matches a regular expression. +- `if: $CI_COMMIT_BRANCH =~ /regex-expression/`: If the commit branch matches a regular expression. - `if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /Merge branch.*/`: If the commit branch is the default branch and the commit message title matches a regular expression. For example, the default commit message for a merge commit starts with `Merge branch`. -- `if: '$CUSTOM_VARIABLE !~ /regex-expression/'`: If the [custom variable](../variables/index.md#custom-cicd-variables) +- `if: $CUSTOM_VARIABLE !~ /regex-expression/`: If the [custom variable](../variables/index.md#custom-cicd-variables) `CUSTOM_VARIABLE` does **not** match a regular expression. -- `if: '$CUSTOM_VARIABLE == "value1"'`: If the custom variable `CUSTOM_VARIABLE` is +- `if: $CUSTOM_VARIABLE == "value1"`: If the custom variable `CUSTOM_VARIABLE` is exactly `value1`. ### Variables in `rules:changes` @@ -643,7 +643,7 @@ timed rollout 10%: To stop the active timer of a delayed job, select **Unschedule** (**{time-out}**). This job can no longer be scheduled to run automatically. You can, however, execute the job manually. -To start a delayed job immediately, select **Play** (**{play}**). +To start a delayed job manually, select **Unschedule** (**{time-out}**) to stop the delay timer and then select **Play** (**{play}**). Soon GitLab Runner starts the job. ## Parallelize large jobs diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index d95199a70d0..b1c4c62c465 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -21,7 +21,7 @@ For advanced CI/CD teams, [custom project templates](../../user/admin_area/custo If you have questions that are not answered here, the [GitLab community forum](https://forum.gitlab.com/) can be a great resource. -## `config.yml` vs `gitlab-ci.yml` +## `config.yml` vs `.gitlab-ci.yml` CircleCI's `config.yml` configuration file defines scripts, jobs, and workflows (known as "stages" in GitLab). In GitLab, a similar approach is used with a `.gitlab-ci.yml` file in the root directory of your repository. @@ -166,7 +166,7 @@ job1: script: - make build rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "try-schedule-workflow"' + - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "try-schedule-workflow" ``` After the pipeline configuration is saved, you configure the cron schedule in the [GitLab UI](../pipelines/schedules.md#add-a-pipeline-schedule), and can enable or disable schedules in the UI as well. @@ -221,7 +221,7 @@ deploy: script: - echo "Deploy job" rules: - - if: '$CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^rc-/' + - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH =~ /^rc-/ ``` ### Caching diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md index 19b1a6cad1f..c59116ea8ed 100644 --- a/doc/ci/migration/jenkins.md +++ b/doc/ci/migration/jenkins.md @@ -305,7 +305,7 @@ my_job: In GitLab, we use the [`variables` keyword](../yaml/index.md#variables) to define different variables at runtime. These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/index.md), -including the section on [protected variables](../variables/index.md#protect-a-cicd-variable) which can be used +including the section on [protected variables](../variables/index.md#protected-cicd-variables) which can be used to limit access to certain variables to certain environments or runners: ```yaml diff --git a/doc/ci/pipeline_editor/index.md b/doc/ci/pipeline_editor/index.md index d72cb14681d..57a9d7a9358 100644 --- a/doc/ci/pipeline_editor/index.md +++ b/doc/ci/pipeline_editor/index.md @@ -19,6 +19,7 @@ From the pipeline editor page you can: - [Validate](#validate-ci-configuration) your configuration syntax while editing the file. - Do a deeper [lint](#lint-ci-configuration) of your configuration, that verifies it with any configuration added with the [`include`](../yaml/index.md#include) keyword. +- View a [list of the CI/CD configuration added with the `include` keyword](#view-included-cicd-configuration). - See a [visualization](#visualize-ci-configuration) of the current configuration. - View an [expanded](#view-expanded-configuration) version of your configuration. - [Commit](#commit-changes-to-ci-configuration) the changes to a specific branch. @@ -50,6 +51,20 @@ reflected in the CI lint. It displays the same results as the existing [CI Lint  +## View included CI/CD configuration + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7064) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `pipeline_editor_file_tree`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) +named `pipeline_editor_file_tree`. + +You can review configuration added with the [`include`](../yaml/index.md#include) +keyword in the pipeline editor. In the top right, select the file tree (**{file-tree}**) +to see a list of all included configuration files. Selected files open in a new tab +for review. + ## Visualize CI configuration > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241722) in GitLab 13.5. diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index f9d9a4f738b..2b18b1d353b 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -90,6 +90,8 @@ To view CI/CD minutes being used for your group: ## View CI/CD minutes used by a personal namespace +> Displaying shared runners duration [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345795) in GitLab 15.0. + You can view the number of CI/CD minutes being used by a personal namespace: 1. On the top bar, in the top right corner, select your avatar. @@ -103,7 +105,7 @@ These additional CI/CD minutes: - Are used only after the monthly quota included in your subscription runs out. - Are carried over to the next month, if any remain at the end of the month. -- Don't expire. +- Are valid for 12 months from date of purchase or until all minutes are consumed, whichever comes first. Expiry of minutes is not currently enforced. If you use more CI/CD minutes than your monthly quota, when you purchase more, those CI/CD minutes are deducted from your quota. For example, with a GitLab SaaS @@ -191,17 +193,41 @@ The cost factor for a job running on a shared runner is: ### Additional costs on GitLab SaaS -On GitLab SaaS, shared runners can have different cost factors depending on the cost involved -in executing the runner. For example, a high spec shared runner could be set to have a cost factor of `2`. -Conversely, a shared runner that executes jobs for public projects could have a low cost factor, like `0.008`. +GitLab SaaS shared runners have different cost factors, depending on the runner type (Linux, Windows, macOS) and the virtual machine configuration. + +| GitLab SaaS runner type | Virtual machine configuration | CI/CD minutes cost factor | +| :--------- | :------------------- | :--------- | +| Linux OS + Docker executor| 1 vCPU, 3.75 GB RAM |1| +| macOS + shell executor | 4 vCPU, 10 GB RAM| 6 | ### Monthly reset of CI/CD minutes On the first day of each calendar month, the accumulated usage of CI/CD minutes is reset to `0` -for all namespaces that use shared runners. +for all namespaces that use shared runners. This means your full quota is available, and +calculations start again from `0`. + +For example, if you have a monthly quota of `10,000` CI/CD minutes: + +- On **1st April**, you have `10,000` minutes. +- During April, you use only `6,000` of the `10,000` minutes. +- On **1st May**, the accumulated usage of minutes resets to `0`, and you have `10,000` minutes to use again + during May. Usage data for the previous month is kept to show historical view of the consumption over time. +### Monthly rollover of purchased CI/CD minutes + +If you purchase additional CI/CD minutes and don't use the full amount, the remaining amount rolls over to +the next month. + +For example: + +- On **1st April**, you purchase `5,000` CI/CD minutes. +- During April, you use only `3,000` of the `5,000` minutes. +- On **1st May**, the remaining `2,000` minutes roll over and are added to your monthly quota. + +Additional CI/CD minutes are a one-time purchase and do not renew or refresh each month. + ## What happens when you exceed the quota When the quota of CI/CD minutes is used for the current month, GitLab stops diff --git a/doc/ci/pipelines/img/downstream_pipeline_actions.png b/doc/ci/pipelines/img/downstream_pipeline_actions.png Binary files differnew file mode 100644 index 00000000000..4c4384bab57 --- /dev/null +++ b/doc/ci/pipelines/img/downstream_pipeline_actions.png diff --git a/doc/ci/pipelines/img/multi_pipeline_mini_graph.gif b/doc/ci/pipelines/img/multi_pipeline_mini_graph.gif Binary files differdeleted file mode 100644 index de49ba5aa12..00000000000 --- a/doc/ci/pipelines/img/multi_pipeline_mini_graph.gif +++ /dev/null diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index 4d1af735b13..c6142ebefc5 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -449,6 +449,22 @@ Pipeline analytics are available on the [**CI/CD Analytics** page](../../user/an Pipeline status and test coverage report badges are available and configurable for each project. For information on adding pipeline badges to projects, see [Pipeline badges](settings.md#pipeline-badges). +### Downstream pipelines + +> Cancel or retry downstream pipelines from the graph view [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354974) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `downstream_retry_action`. Disabled by default. + +In the pipeline graph view, downstream pipelines ([Multi-project pipelines](multi_project_pipelines.md) +and [Parent-child pipelines](parent_child_pipelines.md)) display as a list of cards +on the right of the graph. + +To cancel a downstream pipeline that is still running, select **Cancel** (**{cancel}**) +on the pipeline's card. + +To retry a failed downstream pipeline, select **Retry** (**{retry}**) +on the pipeline's card. + + + ## Pipelines API GitLab provides API endpoints to: diff --git a/doc/ci/pipelines/merge_request_pipelines.md b/doc/ci/pipelines/merge_request_pipelines.md index 75408de2721..89839de718b 100644 --- a/doc/ci/pipelines/merge_request_pipelines.md +++ b/doc/ci/pipelines/merge_request_pipelines.md @@ -20,7 +20,7 @@ Branch pipelines: - Run when you push a new commit to a branch. - Are the default type of pipeline. - Have access to [some predefined variables](../variables/predefined_variables.md). -- Have access to [protected variables](../variables/index.md#protect-a-cicd-variable) and [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). +- Have access to [protected variables](../variables/index.md#protected-cicd-variables) and [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). Merge request pipelines: @@ -33,7 +33,7 @@ Merge request pipelines: - Do not run by default. The jobs in the CI/CD configuration file [must be configured](#prerequisites) to run in merge request pipelines. - Have access to [more predefined variables](#available-predefined-variables). -- Do not have access to [protected variables](../variables/index.md#protect-a-cicd-variable) or [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). +- Do not have access to [protected variables](../variables/index.md#protected-cicd-variables) or [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). Both of these types of pipelines can appear on the **Pipelines** tab of a merge request. @@ -127,12 +127,12 @@ Pipelines for forks display with the **fork** badge in the parent project: ### Run pipelines in the parent project **(PREMIUM)** -Project members in the parent project can choose to run a merge request pipeline +Project members in the parent project can trigger a merge request pipeline for a merge request submitted from a fork project. This pipeline: - Is created and runs in the parent (target) project, not the fork (source) project. -- Uses the CI/CD configuration present in the fork project's branch -- Uses the parent project's CI/CD configuration, resources, and project CI/CD variables. +- Uses the CI/CD configuration present in the fork project's branch. +- Uses the parent project's CI/CD settings, resources, and project CI/CD variables. - Uses the permissions of the parent project member that triggers the pipeline. Run pipelines in fork project MRs to ensure that the post-merge pipeline passes in @@ -142,8 +142,12 @@ running the pipeline in the parent project uses the parent project's trusted run WARNING: Fork merge requests can contain malicious code that tries to steal secrets in the parent project when the pipeline runs, even before merge. As a reviewer, carefully -check the changes in the merge request before triggering the pipeline. GitLab shows -a warning that you must accept before you can trigger the pipeline. +check the changes in the merge request before triggering the pipeline. If you trigger +the pipeline by selecting **Run pipeline** or applying a suggestion, GitLab shows +a warning that you must accept before the pipeline runs. If you trigger the pipeline +by using any other method, including the API, [`/rebase` quick action](../../user/project/quick_actions.md#issues-merge-requests-and-epics), +or [**Rebase** option](../../user/project/merge_requests/methods/index.md#rebasing-in-semi-linear-merge-methods), +**no warning displays**. Prerequisites: @@ -152,10 +156,10 @@ Prerequisites: user running the pipeline. Otherwise, the **Pipelines** tab does not display in the merge request. -To run a pipeline in the parent project for a merge request from a fork project: +To use the UI to run a pipeline in the parent project for a merge request from a fork project: 1. In the merge request, go to the **Pipelines** tab. -1. Select **Run pipeline**. You must accept the warning, or the pipeline does not run. +1. Select **Run pipeline**. You must read and accept the warning, or the pipeline does not run. ## Available predefined variables diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index 12ea3a70536..d200dde143c 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -24,7 +24,7 @@ requests, each waiting to be merged into the target branch. Many merge requests can be added to the train. Each merge request runs its own merged results pipeline, which includes the changes from all of the other merge requests in *front* of it on the train. -All the pipelines run in parallel, to save time. +All the pipelines run in parallel, to save time. The author of the internal merged result commit is always the user that initiated the merge. If the pipeline for a merge request fails, the breaking changes are not merged, and the target branch is unaffected. The merge request is removed from the train, and all pipelines behind it restart. @@ -73,6 +73,9 @@ To enable merge trains: - Your repository must be a GitLab repository, not an [external repository](../ci_cd_for_external_repos/index.md). +Merge trains do not work with [Semi-linear history merge requests](../../user/project/merge_requests/methods/index.md#merge-commit-with-semi-linear-history) +or [fast-forward merge requests](../../user/project/merge_requests/methods/index.md#fast-forward-merge). + ## Enable merge trains To enable merge trains for your project: @@ -84,7 +87,6 @@ To enable merge trains for your project: 1. On the left sidebar, select **Settings > General**. 1. Expand **Merge requests**. 1. In the **Merge method** section, verify that **Merge commit** is selected. - You cannot use **Merge commit with semi-linear history** or **Fast-forward merge** with merge trains. 1. In the **Merge options** section, select **Enable merged results pipelines** (if not already selected) and **Enable merge trains**. 1. Select **Save changes**. diff --git a/doc/ci/pipelines/merged_results_pipelines.md b/doc/ci/pipelines/merged_results_pipelines.md index 7df9ea3f72f..9661d2f5263 100644 --- a/doc/ci/pipelines/merged_results_pipelines.md +++ b/doc/ci/pipelines/merged_results_pipelines.md @@ -12,7 +12,8 @@ A *merged results pipeline* is a type of [merge request pipeline](merge_request_ GitLab creates an internal commit with the merged results, so the pipeline can run against it. This commit does not exist in either branch, -but you can view it in the pipeline details. +but you can view it in the pipeline details. The author of the internal commit is +always the user that created the merge request. The pipeline runs against the target branch as it exists at the moment you run the pipeline. Over time, while you're working in the source branch, the target branch might change. @@ -35,7 +36,7 @@ To use merged results pipelines: - Your repository must be a GitLab repository, not an [external repository](../ci_cd_for_external_repos/index.md). - You must not be using [fast forward merges](../../user/project/merge_requests/fast_forward_merge.md). - [An issue exits](https://gitlab.com/gitlab-org/gitlab/-/issues/26996) to change this behavior. + [An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/26996) to change this behavior. ## Enable merged results pipelines diff --git a/doc/ci/pipelines/multi_project_pipelines.md b/doc/ci/pipelines/multi_project_pipelines.md index e6af9292fe1..20184635e4a 100644 --- a/doc/ci/pipelines/multi_project_pipelines.md +++ b/doc/ci/pipelines/multi_project_pipelines.md @@ -308,7 +308,10 @@ When you configure GitLab CI/CD for your project, you can visualize the stages o  -In the merge request, on the **Pipelines** tab, multi-project pipeline mini-graphs are displayed. -They expand and are shown adjacent to each other when hovering (or tapping on touchscreen devices). +## Retry or cancel multi-project pipelines - +If you have permission to trigger pipelines in the downstream project, you can +retry or cancel multi-project pipelines: + +- [In the main graph view](index.md#downstream-pipelines). +- From the downstream pipeline's details page. diff --git a/doc/ci/pipelines/parent_child_pipelines.md b/doc/ci/pipelines/parent_child_pipelines.md index a3ded24e8c9..3206345d757 100644 --- a/doc/ci/pipelines/parent_child_pipelines.md +++ b/doc/ci/pipelines/parent_child_pipelines.md @@ -216,3 +216,10 @@ multi-project pipelines: - [By using the `variable` keyword](multi_project_pipelines.md#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword). - [By using variable inheritance](multi_project_pipelines.md#pass-cicd-variables-to-a-downstream-pipeline-by-using-variable-inheritance). + +## Retry or cancel child pipelines + +You can retry or cancel child pipelines: + +- [In the main graph view](index.md#downstream-pipelines). +- In the child pipeline's details page. diff --git a/doc/ci/pipelines/pipeline_artifacts.md b/doc/ci/pipelines/pipeline_artifacts.md index e9dd1b2a942..3a1367f4a8c 100644 --- a/doc/ci/pipelines/pipeline_artifacts.md +++ b/doc/ci/pipelines/pipeline_artifacts.md @@ -11,7 +11,7 @@ different to [job artifacts](job_artifacts.md) because they are not explicitly m `.gitlab-ci.yml` definitions. Pipeline artifacts are used by the [test coverage visualization feature](../../user/project/merge_requests/test_coverage_visualization.md) -to collect coverage information. It uses the [`artifacts: reports`](../yaml/index.md#artifactsreports) CI/CD keyword. +to collect coverage information. ## Storage diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index b6ea16ae224..2a95a4e1421 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -222,36 +222,49 @@ averaged. To add test coverage results to a merge request using the project's `.gitlab-ci.yml` file, provide a regular expression using the [`coverage`](../yaml/index.md#coverage) keyword. -Setting the regular expression this way takes precedence over project settings. +<!-- start_remove The following content will be removed on remove_date: '2023-08-22' --> -### Add test coverage results using project settings (DEPRECATED) +### Add test coverage results using project settings (removed) -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.9. Replaced by [`coverage` keyword](../yaml/index.md#coverage). +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.8. Replaced by [`coverage` keyword](../yaml/index.md#coverage). +> [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) -in GitLab 14.9, and is planned for [removal](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. +This feature is in its end-of-life process. It was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) +in GitLab 14.8. The feature is [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. -You can add test coverage results to merge requests using the Project's CI/CD settings: +To migrate from a project setting to the `coverage` keyword, add the [former project setting](#locate-former-project-setting) +to a CI/CD job. For example: -- Set using the GitLab UI: +- A Go test coverage project setting: `coverage: \d+.\d+% of statements`. +- A CI/CD job with `coverage` keyword setting: - 1. On the top bar, select **Menu > Projects** and find your project. - 1. On the left sidebar, select **Settings > CI/CD**. - 1. Expand **General pipelines**. - 1. In the **Test coverage parsing** field, enter a regular expression. Leave blank to disable this feature. + ```yaml + unit-test: + stage: test + coverage: '/coverage: \d+.\d+% of statements/' + script: + - go test -cover + ``` -- Set when [editing a project](../../api/projects.md#edit-project) or [creating a project](../../api/projects.md#create-project) - using the GitLab API with the `build_coverage_regex` attribute: +The `.gitlab-ci.yml` job [`coverage`](../yaml/index.md#coverage) keyword must be: - ```shell - curl --request PUT --header "PRIVATE-TOKEN: <your-token>" \ - --url 'https://gitlab.com/api/v4/projects/<your-project-ID>' \ - --data "build_coverage_regex=<your-regular-expression>" - ``` +- A regular expression starts and ends with the `/` character. +- Defined as single-quoted string. + +You can verify correct syntax using the [pipeline editor](../pipeline_editor/index.md). + +#### Locate former project setting + +To migrate from the project coverage setting to the `coverage` keyword, use the +regular expression displayed in the settings. Available in GitLab 14.10 and earlier: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **General pipelines**. + +The regular expression you need is in the **Test coverage parsing** field. -You can use <https://rubular.com> to test your regular expression. The regular expression returns the **last** -match found in the output. +<!-- end_remove --> ### Test coverage examples @@ -266,6 +279,7 @@ Use this regex for commonly used test tools. - gcovr (C/C++). Example: `^TOTAL.*\s+(\d+\%)$`. - `tap --coverage-report=text-summary` (NodeJS). Example: `^Statements\s*:\s*([^%]+)`. - `nyc npm test` (NodeJS). Example: `All files[^|]*\|[^|]*\s+([\d\.]+)`. +- `jest --ci --coverage` (NodeJS). Example: `All files[^|]*\|[^|]*\s+([\d\.]+)`. - excoveralls (Elixir). Example: `\[TOTAL\]\s+(\d+\.\d+)%`. - `mix test --cover` (Elixir). Example: `\d+.\d+\%\s+\|\s+Total`. - JaCoCo (Java/Kotlin). Example: `Total.*?([0-9]{1,3})%`. diff --git a/doc/ci/quick_start/index.md b/doc/ci/quick_start/index.md index fbdf226181b..e35a042a320 100644 --- a/doc/ci/quick_start/index.md +++ b/doc/ci/quick_start/index.md @@ -19,7 +19,7 @@ If you are migrating from another CI/CD tool, view this documentation: - [Migrate from CircleCI](../migration/circleci.md). - [Migrate from Jenkins](../migration/jenkins.md). -> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch [First time GitLab & CI/CD](https://www.youtube.com/watch?v=kTNfi5z6Uvk&t=553s). This includes a quick introduction to GitLab, the first steps with CI/CD, building a Go project, running tests, using the CI/CD pipeline editor, detecting secrets and security vulnerabilities and offers more exercises for async practice. +> - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch [First time GitLab & CI/CD](https://www.youtube.com/watch?v=kTNfi5z6Uvk&t=553s). This includes a quick introduction to GitLab, the first steps with CI/CD, building a Go project, running tests, using the CI/CD pipeline editor, detecting secrets and security vulnerabilities and offers more exercises for asynchronous practice. > - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch [Intro to GitLab CI](https://www.youtube.com/watch?v=l5705U8s_nQ&t=358s). This workshop uses the Web IDE to quickly get going with building source code using CI/CD, and run unit tests. ## CI/CD process overview @@ -27,6 +27,8 @@ If you are migrating from another CI/CD tool, view this documentation: To use GitLab CI/CD: 1. [Ensure you have runners available](#ensure-you-have-runners-available) to run your jobs. + GitLab SaaS provides runners, so if you're using GitLab.com, you can skip this step. + If you don't have a runner, [install GitLab Runner](https://docs.gitlab.com/runner/install/) and [register a runner](https://docs.gitlab.com/runner/register/) for your instance, project, or group. 1. [Create a `.gitlab-ci.yml` file](#create-a-gitlab-ciyml-file) @@ -130,7 +132,7 @@ The pipeline starts when the commit is committed. ```yaml default: - image: ruby:2.7.4 + image: ruby:2.7.5 ``` This command tells the runner to use a Ruby image from Docker Hub diff --git a/doc/ci/resource_groups/index.md b/doc/ci/resource_groups/index.md index 9312f4a8850..e76c4621a0c 100644 --- a/doc/ci/resource_groups/index.md +++ b/doc/ci/resource_groups/index.md @@ -42,7 +42,7 @@ In this case, the `deploy` jobs across different pipelines could run concurrentl to the `production` environment. Running multiple deployment scripts to the same infrastructure could harm/confuse the instance and leave it in a corrupted state in the worst case. -In order to ensure that a `deploy` job runs once at a time, you can specify +To ensure that a `deploy` job runs once at a time, you can specify [`resource_group` keyword](../yaml/index.md#resource_group) to the concurrency sensitive job: ```yaml @@ -74,27 +74,27 @@ You can choose a process mode to strategically control the job concurrency for y The following modes are supported: - **Unordered:** This is the default process mode that limits the concurrency on running jobs. - It's the easiest option to use and useful when you don't care about the execution order + It's the easiest option to use when you don't care about the execution order of the jobs. It starts processing the jobs whenever a job ready to run. - **Oldest first:** This process mode limits the concurrency of the jobs. When a resource is free, it picks the first job from the list of upcoming jobs (`created`, `scheduled`, or `waiting_for_resource` state) that are sorted by pipeline ID in ascending order. - This mode is useful when you want to ensure that the jobs are executed from the oldest pipeline. - This is less efficient compared to the `unordered` mode in terms of the pipeline efficiency, + This mode is efficient when you want to ensure that the jobs are executed from the oldest pipeline. + It is less efficient compared to the `unordered` mode in terms of the pipeline efficiency, but safer for continuous deployments. - **Newest first:** This process mode limits the concurrency of the jobs. When a resource is free, it picks the first job from the list of upcoming jobs (`created`, `scheduled` or `waiting_for_resource` state) that are sorted by pipeline ID in descending order. - This mode is useful when you want to ensure that the jobs are executed from the newest pipeline and + This mode is efficient when you want to ensure that the jobs are executed from the newest pipeline and cancel all of the old deploy jobs with the [skip outdated deployment jobs](../environments/deployment_safety.md#skip-outdated-deployment-jobs) feature. This is the most efficient option in terms of the pipeline efficiency, but you must ensure that each deployment job is idempotent. ### Change the process mode -To change the process mode of a resource group, you need to use the API and +To change the process mode of a resource group, you must use the API and send a request to [edit an existing resource group](../../api/resource_groups.md#edit-an-existing-resource-group) by specifying the `process_mode`: @@ -145,7 +145,7 @@ Depending on the process mode of the resource group: You can define `resource_group` for downstream pipelines that are sensitive to concurrent executions. The [`trigger` keyword](../yaml/index.md#trigger) can trigger downstream pipelines and the -[`resource_group` keyword](../yaml/index.md#resource_group) can co-exist with it. `resource_group` is useful to control the +[`resource_group` keyword](../yaml/index.md#resource_group) can co-exist with it. `resource_group` is efficient to control the concurrency of deployment pipelines, while other jobs can continue to run concurrently. The following example has two pipeline configurations in a project. When a pipeline starts running, @@ -205,7 +205,7 @@ Read more how you can use GitLab for [safe deployments](../environments/deployme ### Avoid dead locks in pipeline configurations -Since [`oldest_first` process mode](#process-modes) enforces the jobs to be executed in a pipeline order, +Because [`oldest_first` process mode](#process-modes) enforces the jobs to be executed in a pipeline order, there is a case that it doesn't work well with the other CI features. For example, when you run [a child pipeline](../pipelines/parent_child_pipelines.md) @@ -229,10 +229,10 @@ deploy: In a parent pipeline, it runs the `test` job that subsequently runs a child pipeline, and the [`strategy: depend` option](../yaml/index.md#triggerstrategy) makes the `test` job wait until the child pipeline has finished. The parent pipeline runs the `deploy` job in the next stage, that requires a resource from the `production` resource group. -If the process mode is `oldest_first`, it executes the jobs from the oldest pipelines, meaning the `deploy` job is going to be executed next. +If the process mode is `oldest_first`, it executes the jobs from the oldest pipelines, meaning the `deploy` job is executed next. However, a child pipeline also requires a resource from the `production` resource group. -Since the child pipeline is newer than the parent pipeline, the child pipeline +Because the child pipeline is newer than the parent pipeline, the child pipeline waits until the `deploy` job is finished, something that will never happen. In this case, you should specify the `resource_group` keyword in the parent pipeline configuration instead: diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index 6b14cea51d6..e7165339ea0 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -148,7 +148,7 @@ different places. ### Determine the IP address of a shared runner -To view the IP address of a shared runner you must have admin access to +To view the IP address of a shared runner you must have administrator access to the GitLab instance. To determine this: 1. On the top bar, select **Menu > Admin**. @@ -302,6 +302,9 @@ globally or for individual jobs: You can also use variables to configure how many times a runner [attempts certain stages of job execution](#job-stages-attempts). +When using the Kubernetes executor, you can use variables to +[override Kubernetes CPU and memory allocations for requests and limits](https://docs.gitlab.com/runner/executors/kubernetes.html#overwriting-container-resources). + ### Git strategy > - Introduced in GitLab 8.9 as an experimental feature. diff --git a/doc/ci/runners/saas/macos/codesigning.md b/doc/ci/runners/saas/macos/codesigning.md new file mode 100644 index 00000000000..4f8316faf17 --- /dev/null +++ b/doc/ci/runners/saas/macos/codesigning.md @@ -0,0 +1,121 @@ +--- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Code signing for SaaS runners on macOS + +> Introduced in GitLab 15.0. + +Before you can integrate GitLab with Apple services, install to a device, or deploy to the Apple App Store, you must [code sign](https://developer.apple.com/support/code-signing/) your application. + +To code sign an iOS project, you need the following files: + +- A certifcate issued by Apple. +- A provisioning profile. + +## Code signing iOS Projects with fastlane + +When you use SaaS runners on macOS, each job runs on a VM. Included in each VM is [fastlane](https://fastlane.tools/), +an open-source solution aimed at simplifying mobile app deployment. + +These steps outline the minimal setup required to use fastlane to code sign your application. Refer to the fastlane [getting started guide](https://docs.fastlane.tools/), [best practices for integrating with GitLab CI](https://docs.fastlane.tools/best-practices/continuous-integration/gitlab/) and the [fastlane code signing getting started guide](https://docs.fastlane.tools/codesigning/getting-started/) for installation instructions, and an overview of how to use fastlane to handle code signing. + +To use fastlane to code sign your application: + +1. At the root of your project repository, on your local development system, run this command: + + ```plaintext + fastlane match init + ``` + + This command creates the `fastlane` directory and adds two files: `Fastfile` and `Appfile`. + +1. Open `Appfile` and edit it to include your Apple ID and app ID. + + ```plaintext + app_identifier("APP IDENTIFIER") # The bundle identifier of your app + + apple_id("APPLE ID") # Your Apple email address + ``` + +1. Open `Fastfile`, which includes the fastlane build steps. + In the following snippet, the steps `get_certificates`, `get_provisioning_profile,match`, `gym`, and + `upload_to_testflight` are fastlane [actions](https://docs.fastlane.tools/actions/). + + ```plaintext + # This file contains the fastlane.tools configuration + # You can find the documentation at https://docs.fastlane.tools + + default_platform(:ios) + + platform :ios do + desc "Build the application" + lane :beta do + increment_build_number( + build_number: latest_testflight_build_number + 1, + xcodeproj: "${PROJECT_NAME}.xcodeproj" + ) + get_certificates + get_provisioning_profile + # match(type: "appstore",read_only: true) + gym + upload_to_testflight + end + end + ``` + +The example configuration also includes an optional `Gymfile`. This file stores configuration +parameters and is used by the fastlane [`gym`](https://docs.fastlane.tools/actions/gym/) action. + +## Using fastlane match + +To simplify the code signing process and implement the +[Code Signing Best Practices Guide](https://codesigning.guide/) recommendations, +use [fastlane match](https://docs.fastlane.tools/actions/match/). + +- Use one code signing identity shared across your team. +- Store the required certificates and provisioning profiles in a separate GitLab project repository. + +Match automatically syncs iOS and macOS keys and provisioning profiles across all team members with access to the GitLab project. Each team member with access to the project can use the credentials for code signing. + +To use fastlane match: + +1. Initialize match in the project repository: + + ```shell + bundle exec fastlane match init + ``` + +1. Select `git` as your storage node. +1. Enter the URL of the GitLab project you plan to use to store your code signing identities. +1. Optional. To create a new certificate and provisioning profile, run: + + ```shell + bundle exec fastlane match development + ``` + +For different code signing identities' storage options, and for a complete step-by-step guide for using match, +refer to the [match documentation](https://docs.fastlane.tools/actions/match/#usage). + +### Environment variables and authentication + +To complete the setup, you must configure environment variables to use with fastlane. The required variables are outlined in the [fastlane documentation](https://docs.fastlane.tools/best-practices/continuous-integration/#environment-variables-to-set). + +To support Apple's two factor authentication requirement, configure these variables: + +- `FASTLANE_APPLE_APPLICATION_SPECIFIC_PASSWORD` and +- `FASTLANE_SESSION` + +To authenticate fastlane with the App Store for the TestFlight upload, configure these variables: + +- `FASTLANE_USER` and +- `FASTLANE_PASSWORD` + +View the [fastlane authentication with Apple Services guide](https://docs.fastlane.tools/getting-started/ios/authentication/) for an overview of authentication options. + +## Related topics + +- [Apple Developer Support - Code Signing](https://developer.apple.com/support/code-signing/) +- [Code Signing Best Practice Guide](https://codesigning.guide/) diff --git a/doc/ci/runners/saas/macos/environment.md b/doc/ci/runners/saas/macos/environment.md index 4d209fe4cd6..edd897e4c23 100644 --- a/doc/ci/runners/saas/macos/environment.md +++ b/doc/ci/runners/saas/macos/environment.md @@ -9,12 +9,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w When you use SaaS runners on macOS: - Each of your jobs runs in a newly provisioned VM, which is dedicated to the specific job. -- The VM is active only for the duration of the job and immediately deleted. +- The VM is active only for the duration of the job and immediately deleted. This means that any changes that your job makes to the virtual machine will not be available to a subsequent job. +- The virtual machine where your job runs has `sudo` access with no password. + +NOTE: +Each time you run a job that requires tooling or dependencies not available in the base image, those items must be added to the newly provisioned build VM. That process will likely increase the total job duration. ## VM types -The virtual machine where your job runs has `sudo` access with no password. -For the [Beta](../../../../policy/alpha-beta-support.md#beta-features), there is only one available machine type, `gbc-macos-large`. +GitLab SaaS provides macOS build machines on Apple servers with Intel x86-64 processors. +The expectation is that virtual machines running on the Apple M1 chip will be available in the second half of 2022. + +At this time there is only one available machine type offered, `gbc-macos-large`. | Instance type | vCPUS | Memory (GB) | | --------- | --- | ------- | @@ -22,23 +28,34 @@ For the [Beta](../../../../policy/alpha-beta-support.md#beta-features), there is ## VM images +### Image update policy + +GitLab expects to release new images based on this cadence: + +macOS updates: + +- **For new OS versions:** When Apple releases a new macOS version to developers (like macOS `12`), GitLab will plan to release an image based on the OS within the next 30 business days. The image is considered `beta` and the contents of the image (including tool versions) are subject to change until the first patch release (`12.1`). The long-term name will not include `beta` (for example, `macos-12-xcode-13`), so customers are moved automatically out of beta over time. GitLab will try to minimize breaking changes between the first two minor versions but makes no guarantees. Tooling often gets critical bug fixes after the first public release of an OS version. + +- **After the first patch release (`12.1`):** + - The image moves to `maintenance` mode. The tools GitLab builds into the image with Homebrew and asdf are frozen. GitLab continues making Xcode updates, security updates, and any non-breaking changes deemed necessary. + - The image for the previous OS version (`11`) moves to `frozen` mode. GitLab then does only unavoidable changes: security updates, runner version upgrades, and setting the production password. + +Both macOS and Xcode follow a yearly release cadence. As time goes on, GitLab increments their versions synchronously (meaning we build macOS 11 with Xcode 12, macOS 12 with Xcode 13, and so on). + +### Available images + You can execute your build on one of the following images. You specify this image in your `.gitlab-ci.yml` file. Each image is running a specific version of macOS and Xcode. -| VM image | Included software | -|---------------------------|--------------------| -| macos-10.13-xcode-7 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | -| macos-10.13-xcode-8 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | -| macos-10.13-xcode-9 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | -| macos-10.14-xcode-10 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/mojave.yml> | -| macos-10.15-xcode-11 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/catalina.yml> | -| macos-11-xcode-12 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/big-sur.yml> | -| macos-11-xcode-13 | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/monterey.yml> - -### Image update policy - -- Support for new macOS versions is planned. -- Additional details on the support policy and image update release process are documented - [in this project](https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/55bf59c8fa88712960afff2bf6ecc5daa879a8f5/docs/overview.md#os-images). +| VM image | Status | Included software | +|---------------------------|--------|--------------------| +| `macos-10.13-xcode-7` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | +| `macos-10.13-xcode-8` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | +| `macos-10.13-xcode-9` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/high-sierra.yml> | +| `macos-10.14-xcode-10` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/mojave.yml> | +| `macos-10.15-xcode-11` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/catalina.yml> | +| `macos-11-xcode-12` | `frozen` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/big-sur.yml> | +| `macos-12-xcode-13` | `maintenance` | <https://gitlab.com/gitlab-org/ci-cd/shared-runners/images/macstadium/orka/-/blob/main/toolchain/monterey.yml> | +| (none, awaiting macOS 13) | `beta` | | diff --git a/doc/ci/runners/saas/macos_saas_runner.md b/doc/ci/runners/saas/macos_saas_runner.md index bad9da960b2..0ab2de36f10 100644 --- a/doc/ci/runners/saas/macos_saas_runner.md +++ b/doc/ci/runners/saas/macos_saas_runner.md @@ -4,10 +4,9 @@ group: Runner info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# SaaS runners on macOS (beta) **(PREMIUM SAAS)** +# SaaS runners on macOS (Limited Availability) **(PREMIUM SAAS)** -SaaS runners on macOS are in [Beta](../../../policy/alpha-beta-support.md#beta-features) -and shouldn't be relied upon for mission-critical production jobs. +SaaS runners on macOS are now in [Limited Availability](../../../policy/alpha-beta-support.md#beta-features) for approved open source programs and customers in Premium and Ultimate plans. SaaS runners on macOS provide an on-demand macOS build environment integrated with GitLab SaaS [CI/CD](../../../ci/index.md). @@ -15,11 +14,17 @@ Use these runners to build, test, and deploy apps for the Apple ecosystem (macOS of all the capabilities of the GitLab single DevOps platform and not have to manage or operate a build environment. +CI/CD minutes used on GitLab SaaS macOS runners are included in your CI/CD minute consumption totals. CI jobs that run on macOS **will** consume CI minutes at a faster rate than CI jobs on the GitLab SaaS runners on Linux. + +Refer to the CI/CD minutes [cost factor](../../../ci/pipelines/cicd_minutes.md#cost-factor) for the cost factor applied to the GitLab SaaS macOS runners. + ## Quickstart -To start using SaaS runners on macOS, you must submit an access request [issue](https://gitlab.com/gitlab-com/macos-buildcloud-runners-beta/-/issues/new?issuable_template=beta_access_request). After your -access has been granted and your build environment configured, you must configure your -`.gitlab-ci.yml` pipeline file: +To start using SaaS runners on macOS, you must be an active GitLab SaaS Premium or Ultimate customer. Participants in the GitLab Open Source program are also eligible to use the service. + +### Configuring your pipeline + +To start using the SaaS runners on macOS to run your CI jobs, you must configure your `.gitlab-ci.yml` file: 1. Add a `.gitlab-ci.yml` file to your project repository. 1. Specify the [image](macos/environment.md#vm-images) you want to use. @@ -27,7 +32,7 @@ access has been granted and your build environment configured, you must configur The runners automatically run your build. -## Example `.gitlab-ci.yml` file +### Example `.gitlab-ci.yml` file The following sample `.gitlab-ci.yml` file shows how to start using the SaaS runners on macOS: @@ -42,7 +47,7 @@ stages: - test before_script: - - echo "started by ${GITLAB_USER_NAME}" + - echo "started by ${GITLAB_USER_NAME}" build: extends: @@ -60,4 +65,13 @@ test: ``` NOTE: -During the Beta period, the architecture of this solution will change. Rather than the jobs running on a specific VM instance, they will run on an ephemeral VM instance that is created by an autoscaling instance, known as the Runner Manager. We will notify all Beta participants of any downtime required to do this work. +You can specify a different Xcode image to run a job. To do so, replace the value for the `image` keyword with the value of the [virtual machine image name](macos/environment.md#vm-images) from the list of available images. + +## SaaS runners on macOS service level objective + +In SaaS runners on macOS, the objective is to make 90% of CI jobs start executing in 120 seconds or less. The error rate should be less than 0.5%. + +## Known Limitations and Usage Constraints + +- If the VM image does not include the specific software version you need for your job, then the job execution time will increase as the required software needs to be fetched and installed. +- At this time, it is not possible to bring your own OS image. diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index 3cc38a8b74c..2bf360e69f1 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -47,7 +47,6 @@ The response returns all of the metadata for the file you just uploaded. For exa "name": "myfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", "checksum_algorithm": "sha256", - "permissions": "read_only", "created_at": "2022-02-22T22:22:22.222Z" } ``` @@ -71,7 +70,6 @@ The response returns an array of all of the secure files in the project. For exa "name": "myfile.jks", "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", "checksum_algorithm": "sha256", - "permissions": "read_only", "created_at": "2022-02-22T22:22:22.222Z" }] ``` diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index e6406818b4c..e876c6d7326 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -80,7 +80,7 @@ As mentioned before, this feature is designed to provide **network accessible** services. A database is the simplest example of such a service. The services feature is not designed to, and does not, add any software from the -defined `services` image(s) to the job's container. +defined `services` images to the job's container. For example, if you have the following `services` defined in your job, the `php`, `node` or `go` commands are **not** available for your script, and the job fails: diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 81cb924532c..1757543be5b 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -16,7 +16,7 @@ This guide also lists common issues and possible solutions. An early source of problems can be incorrect syntax. The pipeline shows a `yaml invalid` badge and does not start running if any syntax or formatting problems are found. -### Edit `gitlab-ci.yml` with the pipeline editor +### Edit `.gitlab-ci.yml` with the pipeline editor The [pipeline editor](pipeline_editor/index.md) is the recommended editing experience (rather than the single file editor or the Web IDE). It includes: @@ -331,6 +331,12 @@ busy_resources.pluck(:build_id) busy_resources.update_all(build_id: nil) ``` +### Job log slow to update + +When you visit the job log page for a running job, there could be a delay of up to +60 seconds before the log updates. The default refresh time is 60 seconds, but after +the log is viewed in the UI, the following log updates should occur every 3 seconds. + ## How to get help If you are unable to resolve pipeline issues, you can get help from: diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index 30d53a340a9..90403351b11 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -168,7 +168,7 @@ To add or update variables in the project settings: - **Type**: [`File` or `Variable`](#cicd-variable-types). - **Environment scope**: Optional. `All`, or specific [environments](../environments/index.md). - **Protect variable** Optional. If selected, the variable is only available - in pipelines that run on protected branches or tags. + in pipelines that run on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). - **Mask variable** Optional. If selected, the variable's **Value** is masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#mask-a-cicd-variable). @@ -367,18 +367,21 @@ a large value to the trace log has the potential to be [revealed](https://gitlab When using GitLab Runner 14.2, only the tail of the variable, characters beyond 4KiB in length, have the potential to be revealed. -### Protect a CI/CD variable +### Protected CI/CD variables -You can protect a project, group or instance CI/CD variable so it is only passed -to pipelines running on [protected branches](../../user/project/protected_branches.md) +You can configure a project, group, or instance CI/CD variable to be available +only to pipelines that run on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). -[Merge request pipelines](../pipelines/merge_request_pipelines.md) do not have access to protected variables. -An [issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/28002) regarding this limitation. +[Merged results pipelines](../pipelines/merge_request_pipelines.md#types-of-merge-request-pipelines), which run on a +temporary merge commit, not a branch or tag, do not have access to these variables. + +Pipelines that run directly on the merge request's source branch, with no added merge commit, can access +these variables if the source branch is a protected branch. -To protect a variable: +To mark a variable as protected: -1. Go to **Settings > CI/CD** in the project, group or instance admin area. +1. Go to **Settings > CI/CD** in the project, group or instance Admin Area. 1. Expand the **Variables** section. 1. Next to the variable you want to protect, select **Edit**. 1. Select the **Protect variable** checkbox. @@ -729,7 +732,7 @@ the variable can be available for. To learn more about scoping environments, see [Scoping environments with specs](../environments/index.md#scope-environments-with-specs). To learn more about ensuring CI/CD variables are only exposed in pipelines running from protected -branches or tags, see [Protect a CI/CD Variable](#protect-a-cicd-variable). +branches or tags, see [Protected CI/CD variables](#protected-cicd-variables). ## Deployment variables @@ -848,8 +851,8 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_SERVER_PROTOCOL=https ++ export CI_SERVER_NAME=GitLab ++ CI_SERVER_NAME=GitLab -++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal -++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,pseudonymizer,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal ++ export CI_PROJECT_ID=17893 ++ CI_PROJECT_ID=17893 ++ export CI_PROJECT_NAME=ci-debug-trace diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 3b504b02fae..f8f775a6df4 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -14,8 +14,6 @@ Some variables are only available with more recent versions of [GitLab Runner](h You can [output the values of all variables available for a job](index.md#list-all-environment-variables) with a `script` command. -There are also [Kubernetes-specific deployment variables (deprecated)](../../user/project/clusters/deploy_to_cluster.md#deployment-variables). - There are also a number of [variables you can use to configure runner behavior](../runners/configure_runners.md#configure-runner-behavior-with-variables) globally or for individual jobs. | Variable | GitLab | Runner | Description | diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index b30b13e1ec0..289e8b91104 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -80,32 +80,30 @@ GitLab can display the results of one or more reports in: - The [security dashboard](../../user/application_security/security_dashboard/index.md). - The [Project Vulnerability report](../../user/application_security/vulnerability_report/index.md). -## `artifacts:reports:cobertura` (DEPRECATED) +<!--- start_remove The following content will be removed on remove_date: '2023-08-22' --> + +## `artifacts:reports:cobertura` (removed) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.9. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/348980) in GitLab 15.0. -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab -14.9 and replaced with `artifacts:reports:coverage_report` in 14.10. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.9 and +[removed](https://gitlab.com/gitlab-org/gitlab/-/issues/348980) in GitLab 15.0. Use `artifacts:reports:coverage_report` +instead. -The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). -The collected Cobertura coverage reports upload to GitLab as an artifact. - -GitLab can display the results of one or more reports in the merge request -[diff annotations](../../user/project/merge_requests/test_coverage_visualization.md). - -Cobertura was originally developed for Java, but there are many third-party ports for other languages such as -JavaScript, Python, and Ruby. +<!--- end_remove --> ## `artifacts:reports:coverage_report` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344533) in GitLab 14.9. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344533) in GitLab 14.10. -Use `coverage_report` to collect coverage report in Cobertura format, similar to `artifacts:reports:cobertura`. +Use `coverage_report` to collect coverage report in Cobertura format. -NOTE: -`artifacts:reports:coverage_report` cannot be used at the same time with `artifacts:reports:cobertura`. +The `cobertura` report collects [Cobertura coverage XML files](../../user/project/merge_requests/test_coverage_visualization.md). + +Cobertura was originally developed for Java, but there are many third-party ports for other languages such as +JavaScript, Python, and Ruby. ```yaml artifacts: diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index e5a543e7130..838710834ba 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -344,7 +344,7 @@ You can only use the following rules with `include` (and only with [certain vari include: - local: builds.yml rules: - - if: '$INCLUDE_BUILDS == "true"' + - if: $INCLUDE_BUILDS == "true" - local: deploys.yml rules: - if: $CI_COMMIT_BRANCH == "main" diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index b335e8e8545..eb587db6d5f 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Keyword reference for the `.gitlab-ci.yml` file **(FREE)** +# `.gitlab-ci.yml` keyword reference **(FREE)** This document lists the configuration options for your GitLab `.gitlab-ci.yml` file. @@ -16,7 +16,7 @@ This document lists the configuration options for your GitLab `.gitlab-ci.yml` f When you are editing your `.gitlab-ci.yml` file, you can validate it with the [CI Lint](../lint.md) tool. -If you are editing this page, make sure you follow the [CI/CD YAML reference style guide](../../development/cicd/cicd_reference_documentation_guide.md). +If you are editing content on this page, follow the [instructions for documenting keywords](../../development/cicd/cicd_reference_documentation_guide.md). ## Keywords @@ -561,7 +561,7 @@ This same warning is displayed when: The default value for `allow_failure` is: - `true` for [manual jobs](../jobs/job_control.md#create-a-job-that-must-be-run-manually). -- `false` for manual jobs that also use [`rules`](#rules). +- `false` for jobs that use `when: manual` inside [`rules`](#rules). - `false` in all other cases. **Keyword type**: Job keyword. You can use it only as part of a job. @@ -1350,8 +1350,6 @@ In this example: **Additional details**: -- Coverage regular expressions set in `gitlab-ci.yml` take precedence over coverage regular expression set in the - [GitLab UI](../pipelines/settings.md#add-test-coverage-results-using-project-settings-deprecated). - If there is more than one matched line in the job output, the last line is used (the first result of reverse search). - If there are multiple matches in a single line, the last match is searched @@ -1561,7 +1559,7 @@ environment. #### `environment:action` -Use the `action` keyword to specify jobs that prepare, start, or stop environments. +Use the `action` keyword to specify how the job interacts with the environment. **Keyword type**: Job keyword. You can use it only as part of a job. @@ -1570,8 +1568,10 @@ Use the `action` keyword to specify jobs that prepare, start, or stop environmen | **Value** | **Description** | |:----------|:----------------| | `start` | Default value. Indicates that the job starts the environment. The deployment is created after the job starts. | -| `prepare` | Indicates that the job is only preparing the environment. It does not trigger deployments. [Read more about preparing environments](../environments/index.md#prepare-an-environment-without-creating-a-deployment). | +| `prepare` | Indicates that the job is only preparing the environment. It does not trigger deployments. [Read more about preparing environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | | `stop` | Indicates that the job stops a deployment. For more detail, read [Stop an environment](../environments/index.md#stop-an-environment). | +| `verify` | Indicates that the job is only verifying the environment. It does not trigger deployments. [Read more about verifying environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | +| `access` | Indicates that the job is only accessing the environment. It does not trigger deployments. [Read more about accessing environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | **Example of `environment:action`**: @@ -3071,12 +3071,12 @@ to configure the job behavior, or with [`workflow`](#workflow) to configure the job: script: echo "Hello, Rules!" rules: - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH' + - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH when: never - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/' + - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^feature/ when: manual allow_failure: true - - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ``` **Additional details**: @@ -3090,6 +3090,8 @@ job: - Unlike variables in [`script`](../variables/index.md#use-cicd-variables-in-job-scripts) sections, variables in rules expressions are always formatted as `$VARIABLE`. - You can use `rules:if` with `include` to [conditionally include other configuration files](includes.md#use-rules-with-include). +- In [GitLab 15.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/35438), + variables on the right side of `=~` and `!~` expressions are evaluated as regular expressions. **Related topics**: @@ -3122,7 +3124,7 @@ branch or merge request pipelines. docker build: script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" changes: - Dockerfile when: manual @@ -3203,7 +3205,7 @@ job to run before continuing. job: script: echo "Hello, Rules!" rules: - - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' + - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH when: manual allow_failure: true ``` @@ -3666,6 +3668,10 @@ trigger_job: earlier, using them together causes the error `jobs:#{job-name} when should be on_success, on_failure or always`. - In [GitLab 13.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/197140/), you can view which job triggered a downstream pipeline in the [pipeline graph](../pipelines/index.md#visualize-pipelines). +- [Manual pipeline variables](../variables/index.md#override-a-defined-cicd-variable) + and [scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule) + are not passed to downstream pipelines by default. Use [trigger:forward](#triggerforward) + to forward these variables to downstream pipelines. **Related topics**: @@ -3796,7 +3802,11 @@ deploy_review_job: - All YAML-defined variables are also set to any linked [Docker service containers](../services/index.md). - YAML-defined variables are meant for non-sensitive project configuration. Store sensitive information - in [protected variables](../variables/index.md#protect-a-cicd-variable) or [CI/CD secrets](../secrets/index.md). + in [protected variables](../variables/index.md#protected-cicd-variables) or [CI/CD secrets](../secrets/index.md). +- [Manual pipeline variables](../variables/index.md#override-a-defined-cicd-variable) + and [scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule) + are not passed to downstream pipelines by default. Use [trigger:forward](#triggerforward) + to forward these variables to downstream pipelines. **Related topics**: diff --git a/doc/ci/yaml/workflow.md b/doc/ci/yaml/workflow.md index b46d504edfa..a985db14d08 100644 --- a/doc/ci/yaml/workflow.md +++ b/doc/ci/yaml/workflow.md @@ -38,7 +38,7 @@ workflow: rules: - if: $CI_COMMIT_MESSAGE =~ /-draft$/ when: never - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_PIPELINE_SOURCE == "push" ``` This example has strict rules, and pipelines do **not** run in any other case. @@ -50,9 +50,9 @@ All other pipeline types run. For example: ```yaml workflow: rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: $CI_PIPELINE_SOURCE == "schedule" when: never - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_PIPELINE_SOURCE == "push" when: never - when: always ``` @@ -81,18 +81,20 @@ but does not run pipelines for any other case. It runs: ```yaml workflow: rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS when: never - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH ``` -If the pipeline is triggered by: +If GitLab attempts to trigger: -- A merge request, run a merge request pipeline. For example, a merge request pipeline +- A merge request pipeline, start the pipeline. For example, a merge request pipeline can be triggered by a push to a branch with an associated open merge request. -- A change to a branch, but a merge request is open for that branch, do not run a branch pipeline. -- A change to a branch, but without any open merge requests, run a branch pipeline. +- A branch pipeline, but a merge request is open for that branch, do not run the branch pipeline. + For example, a branch pipeline can be triggered by a change to a branch, an API call, + a scheduled pipeline, and so on. +- A branch pipeline, but there is no merge request open for the branch, run the branch pipeline. You can also add a rule to an existing `workflow` section to switch from branch pipelines to merge request pipelines when a merge request is created. diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index d263d9b5eb5..35dbd80e4d1 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -158,7 +158,7 @@ and should not be used. Some other points to consider: ### Why explicit names are required As Rails is database agnostic, it generates an index name only -from the required options of all indexes: table name and column name(s). +from the required options of all indexes: table name and column names. For example, imagine the following two indexes are created in a migration: ```ruby @@ -173,7 +173,7 @@ Creation of the second index would fail, because Rails would generate the same name for both indexes. This is further complicated by the behavior of the `index_exists?` method. -It considers only the table name, column name(s) and uniqueness specification +It considers only the table name, column names, and uniqueness specification of the index when making a comparison. Consider: ```ruby @@ -284,8 +284,9 @@ production clone. ### Add a migration to create the index synchronously After the index is verified to exist on the production database, create a second -merge request that adds the index synchronously. The synchronous -migration results in a no-op on GitLab.com, but you should still add the +merge request that adds the index synchronously. The schema changes must be +updated and committed to `structure.sql` in this second merge request. +The synchronous migration results in a no-op on GitLab.com, but you should still add the migration as expected for other installations. The below block demonstrates how to create the second migration for the previous asynchronous example. diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 4f27e811b11..f807ed0f85e 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -97,7 +97,7 @@ discussed in [Nullable fields](#nullable-fields). Fields that use the [`feature_flag` property](#feature_flag-property) and the flag is disabled by default are exempt from the deprecation process, and can be removed at any time without notice. -See the [deprecating fields, arguments, and enum values](#deprecating-fields-arguments-and-enum-values) section for how to deprecate items. +See the [deprecating schema items](#deprecating-schema-items) section for how to deprecate items. ## Global IDs @@ -540,19 +540,39 @@ def foo end ``` -## Deprecating fields, arguments, and enum values +## Deprecating schema items The GitLab GraphQL API is versionless, which means we maintain backwards compatibility with older versions of the API with every change. -Rather than removing fields, arguments, or [enum values](#enums), they -must be _deprecated_ instead. +Rather than removing fields, arguments, [enum values](#enums), or [mutations](#mutations), +they must be _deprecated_ instead. The deprecated parts of the schema can then be removed in a future release in accordance with the [GitLab deprecation process](../api/graphql/index.md#deprecation-and-removal-process). -Fields, arguments, and enum values are deprecated using the `deprecated` property. -The value of the property is a `Hash` of: +To deprecate a schema item in GraphQL: + +1. [Create a deprecation issue](#create-a-deprecation-issue) for the item. +1. [Mark the item as deprecated](#mark-the-item-as-deprecated) in the schema. + +See also: + +- [Aliasing and deprecating mutations](#aliasing-and-deprecating-mutations). +- [How to filter Kibana for queries that used deprecated fields](graphql_guide/monitoring.md#queries-that-used-a-deprecated-field). + +### Create a deprecation issue + +Every GraphQL deprecation should have a deprecation issue created [using the `Deprecations` issue template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Deprecations) to track its deprecation and removal. + +Apply these two labels to the deprecation issue: + +- `~GraphQL` +- `~deprecation` + +### Mark the item as deprecated + +Fields, arguments, enum values, and mutations are deprecated using the `deprecated` property. The value of the property is a `Hash` of: - `reason` - Reason for the deprecation. - `milestone` - Milestone that the field was deprecated. @@ -569,7 +589,7 @@ The original `description` of the things being deprecated should be maintained, and should _not_ be updated to mention the deprecation. Instead, the `reason` is appended to the `description`. -### Deprecation reason style guide +#### Deprecation reason style guide Where the reason for deprecation is due to the field, argument, or enum value being replaced, the `reason` must indicate the replacement. For example, the @@ -601,7 +621,7 @@ end If the field, argument, or enum value being deprecated is not being replaced, a descriptive deprecation `reason` should be given. -### Deprecate Global IDs +#### Deprecate Global IDs We use the [`rails/globalid`](https://github.com/rails/globalid) gem to generate and parse Global IDs, so as such they are coupled to model names. When we rename a @@ -698,11 +718,6 @@ aware of the support. The documentation will mention that the old Global ID style is now deprecated. -See also: - -- [Aliasing and deprecating mutations](#aliasing-and-deprecating-mutations). -- [How to filter Kibana for queries that used deprecated fields](graphql_guide/monitoring.md#queries-that-used-a-deprecated-field). - ## Enums GitLab GraphQL enums are defined in `app/graphql/types`. When defining new enums, the @@ -748,7 +763,7 @@ end ``` Enum values can be deprecated using the -[`deprecated` keyword](#deprecating-fields-arguments-and-enum-values). +[`deprecated` keyword](#deprecating-schema-items). ### Defining GraphQL enums dynamically from Rails enums @@ -1713,7 +1728,7 @@ mount_aliased_mutation 'BarMutation', Mutations::FooMutation ``` This allows us to rename a mutation and continue to support the old name, -when coupled with the [`deprecated`](#deprecating-fields-arguments-and-enum-values) +when coupled with the [`deprecated`](#deprecating-schema-items) argument. Example: diff --git a/doc/development/application_slis/index.md b/doc/development/application_slis/index.md index a202bc419e1..2834723fc01 100644 --- a/doc/development/application_slis/index.md +++ b/doc/development/application_slis/index.md @@ -30,18 +30,33 @@ to be emitted from the rails application: ## Defining a new SLI -An SLI can be defined using the `Gitlab::Metrics::Sli` class. +An SLI can be defined using the `Gitlab::Metrics::Sli::Apdex` or +`Gitlab::Metrics::Sli::ErrorRate` class. These work in broadly the same way, but +for clarity, they define different metric names: + +1. `Gitlab::Metrics::Sli::Apdex.new('foo')` defines: + 1. `gitlab_sli:foo_apdex:total` for the total number of measurements. + 1. `gitlab_sli:foo_apdex:success_total` for the number of successful + measurements. +1. `Gitlab::Metrics::Sli::ErrorRate.new('foo')` defines: + 1. `gitlab_sli:foo_error_rate:total` for the total number of measurements. + 1. `gitlab_sli:foo_error_rate:error_total` for the number of error + measurements - as this is an error rate, it's more natural to talk about + errors divided by the total. + +As shown in this example, they can share a base name (`foo` in this example). We +recommend this when they refer to the same operation. Before the first scrape, it is important to have [initialized the SLI with all possible label-combinations](https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics). This avoid confusing results when using these counters in calculations. -To initialize an SLI, use the `.inilialize_sli` class method, for +To initialize an SLI, use the `.initialize_sli` class method, for example: ```ruby -Gitlab::Metrics::Sli.initialize_sli(:received_email, [ +Gitlab::Metrics::Sli::Apdex.initialize_sli(:received_email, [ { feature_category: :team_planning, email_type: :create_issue @@ -67,7 +82,7 @@ this adds is understood and acceptable. Tracking an operation in the newly defined SLI can be done like this: ```ruby -Gitlab::Metrics::Sli[:received_email].increment( +Gitlab::Metrics::Sli::Apdex[:received_email].increment( labels: { feature_category: :service_desk, email_type: :service_desk @@ -79,20 +94,26 @@ Gitlab::Metrics::Sli[:received_email].increment( Calling `#increment` on this SLI will increment the total Prometheus counter ```prometheus -gitlab_sli:received_email:total{ feature_category='service_desk', email_type='service_desk' } +gitlab_sli:received_email_apdex:total{ feature_category='service_desk', email_type='service_desk' } ``` -If the `success:` argument passed is truthy, then the success counter -will also be incremented: +If the `success:` argument passed is truthy, then the success counter will also +be incremented: ```prometheus -gitlab_sli:received_email:success_total{ feature_category='service_desk', email_type='service_desk' } +gitlab_sli:received_email_apdex:success_total{ feature_category='service_desk', email_type='service_desk' } ``` -So far, only tracking `apdex` using a success rate is supported. If you -need to track errors this way, please upvote -[this issue](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1395) -and leave a comment so we can prioritize this. +For error rate SLIs, the equivalent argument is called `error:`: + +```ruby +Gitlab::Metrics::Sli::ErrorRate[:merge].increment( + labels: { + merge_type: :fast_forward + }, + error: !merge_success? +) +``` ## Using the SLI in service monitoring and alerts diff --git a/doc/development/application_slis/rails_request_apdex.md b/doc/development/application_slis/rails_request_apdex.md index 373589aaefc..f9613a14dd1 100644 --- a/doc/development/application_slis/rails_request_apdex.md +++ b/doc/development/application_slis/rails_request_apdex.md @@ -136,8 +136,19 @@ information in the logs to check: 1. The table loads information for the busiest endpoints by default. To speed the response, add both: - - A filter for `json.caller_id.keyword`. - - The identifier you're interested in, such as `Projects::RawController#show`. + + - A filter for `json.meta.caller_id.keyword`. + - The identifier you're interested in, for example: + + ```ruby + Projects::RawController#show + ``` + + or: + + ```plaintext + GET /api/:version/projects/:id/snippets/:snippet_id/raw + ``` 1. Check the [appropriate percentile duration](#request-apdex-slo) for the service handling the endpoint. The overall duration should diff --git a/doc/development/architecture.md b/doc/development/architecture.md index dd432dd5e37..486ef6d27fc 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -347,7 +347,7 @@ Component statuses are linked to configuration documentation for each component. | [Elasticsearch](#elasticsearch) | Improved search within GitLab | ⤓ | ⚙ | ⤓ | ⤓ | ✅ | ⤓ | ⚙ | EE Only | | [Gitaly](#gitaly) | Git RPC service for handling all Git calls made by GitLab | ✅ | ✅ | ✅ | ✅ | ✅ | ⚙ | ✅ | CE & EE | | [GitLab Exporter](#gitlab-exporter) | Generates a variety of GitLab metrics | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | CE & EE | -| [GitLab Geo Node](#gitlab-geo) | Geographically distributed GitLab nodes | ⚙ | ⚙ | ❌ | ❌ | ✅ | ❌ | ⚙ | EE Only | +| [GitLab Geo](#gitlab-geo) | Geographically distributed GitLab site | ⚙ | ⚙ | ❌ | ❌ | ✅ | ❌ | ⚙ | EE Only | | [GitLab Pages](#gitlab-pages) | Hosts static websites | ⚙ | ⚙ | ❌ | ❌ | ✅ | ⚙ | ⚙ | CE & EE | | [GitLab agent](#gitlab-agent) | Integrate Kubernetes clusters in a cloud-native way | ⚙ | ⚙ | ⚙ | ❌ | ❌ | ⤓ | ⚙ | EE Only | | [GitLab self-monitoring: Alertmanager](#alertmanager) | Deduplicates, groups, and routes alerts from Prometheus | ⚙ | ⚙ | ✅ | ⚙ | ✅ | ❌ | ❌ | CE & EE | diff --git a/doc/development/audit_event_guide/index.md b/doc/development/audit_event_guide/index.md index 34f78174e5b..0d62bcdc3b2 100644 --- a/doc/development/audit_event_guide/index.md +++ b/doc/development/audit_event_guide/index.md @@ -25,7 +25,7 @@ To instrument an audit event, the following attributes should be provided: | `scope` | User, Project, Group | true | Scope which the audit event belongs to | | `target` | Object | true | Target object being audited | | `message` | String | true | Message describing the action | -| `created_at` | DateTime | false | The time when the action occured. Defaults to `DateTime.current` | +| `created_at` | DateTime | false | The time when the action occurred. Defaults to `DateTime.current` | ## How to instrument new Audit Events diff --git a/doc/development/backend/create_source_code_be/gitaly_touch_points.md b/doc/development/backend/create_source_code_be/gitaly_touch_points.md new file mode 100644 index 00000000000..5ac362e709f --- /dev/null +++ b/doc/development/backend/create_source_code_be/gitaly_touch_points.md @@ -0,0 +1,27 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Source Code - Gitaly Touch Points + +## RPCs + +Gitaly is a wrapper around the `git` binary, running in a [Gitaly Cluster](../../../administration/gitaly/index.md). It provides managed access to the file system housing the `git` repositories, via Golang Remote Procedure Calls (RPCs). Other functions are access optimization, caching, and a form of pagination against the file system. + +The comprehensive [Beginner's guide to Gitaly contributions](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/beginners_guide.md) is focused on making updates to Gitaly, and offers many insights into how to understand the Gitaly code. + +All access to Gitaly from other parts of GitLab are through Create: Source Code endpoints: + +## The `Commit` model + +After a call is made to Gitaly, Git `commit` information is stored in memory. This information is wrapped by the [Ruby `Commit` Model](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/commit.rb), which is a wrapper around [`Gitlab::Git::Commit`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/git/commit.rb). + +The `Commit` model acts like an ActiveRecord object, but it does not have a PostgreSQL backend. Instead, it maps back to Gitaly RPCs. + +## Rugged Patches + +Historically in GitLab, access to the server-based `git` repositories was provided through the [rugged](https://github.com/libgit2/rugged) RubyGem, which provides Ruby bindings to `libgit2`. This was further extended by what is termed "Rugged Patches", [a set of extensions to the Rugged library](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57317). Rugged implementations of some of the most commonly-used RPCs can be [enabled via feature flags](../../gitaly.md#legacy-rugged-code). + +Rugged access requires the use of a NFS file system, a direction GitLab is moving away from in favor of Gitaly Cluster. Rugged has been proposed for [deprecation and removal](https://gitlab.com/gitlab-org/gitaly/-/issues/1690). Several large customers are still using NFS, and a specific removal date is not planned at this point. diff --git a/doc/development/backend/create_source_code_be/index.md b/doc/development/backend/create_source_code_be/index.md index 8661d8b4d74..ad4e25dc815 100644 --- a/doc/development/backend/create_source_code_be/index.md +++ b/doc/development/backend/create_source_code_be/index.md @@ -35,107 +35,13 @@ for GitLab Shell. ## GitLab Rails -### Source code API endpoints +### Gitaly touch points -| Endpoint | Threshold | Source | -| -----------------------------------------------------------------------------------|---------------------------------------|--------------------------------------------------------------------------------------| -| `DELETE /api/:version/projects/:id/protected_branches/:name` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | -| `GET /api/:version/internal/authorized_keys` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | | | -| `GET /api/:version/internal/lfs` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/lfs.rb)| -| `GET /api/:version/projects/:id/approval_rules` | `:low` | | -| `GET /api/:version/projects/:id/approval_settings` | default | | -| `GET /api/:version/projects/:id/approvals` | default | | -| `GET /api/:version/projects/:id/forks` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | -| `GET /api/:version/projects/:id/groups` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | -| `GET /api/:version/projects/:id/languages` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | -| `GET /api/:version/projects/:id/merge_request_approval_setting` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/merge_request_approval_settings.rb) | -| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_rules` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/merge_request_approval_rules.rb) | -| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_settings` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/project_approval_settings.rb) | -| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_state` | `:low` | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | -| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approvals` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | -| `GET /api/:version/projects/:id/protected_branches` | default |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | -| `GET /api/:version/projects/:id/protected_branches/:name` | default |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | -| `GET /api/:version/projects/:id/protected_tags` | default | | -| `GET /api/:version/projects/:id/protected_tags/:name` | default | | -| `GET /api/:version/projects/:id/push_rule` | default | | -| `GET /api/:version/projects/:id/remote_mirrors` | default | | -| `GET /api/:version/projects/:id/repository/archive` | default | | -| `GET /api/:version/projects/:id/repository/blobs/:sha` | default | | -| `GET /api/:version/projects/:id/repository/blobs/:sha/raw` | default | | -| `GET /api/:version/projects/:id/repository/branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/branches.rb) | -| `GET /api/:version/projects/:id/repository/branches/:branch` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/branches.rb) | -| `GET /api/:version/projects/:id/repository/commits` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| -| `GET /api/:version/projects/:id/repository/commits/:sha` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | -| `GET /api/:version/projects/:id/repository/commits/:sha/comments` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | -| `GET /api/:version/projects/:id/repository/commits/:sha/diff` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | -| `GET /api/:version/projects/:id/repository/commits/:sha/merge_requests` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| -| `GET /api/:version/projects/:id/repository/commits/:sha/refs` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | -| `GET /api/:version/projects/:id/repository/compare` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/repositories.rb) | -| `GET /api/:version/projects/:id/repository/contributors` | default | | -| `GET /api/:version/projects/:id/repository/files/:file_path` | default | | -| `GET /api/:version/projects/:id/repository/files/:file_path/raw` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | -| `GET /api/:version/projects/:id/repository/tags` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/tags.rb) | -| `GET /api/:version/projects/:id/repository/tree` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/repositories.rb) | -| `GET /api/:version/projects/:id/statistics` | default | | -| `GraphqlController#execute` | default | | -| `HEAD /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | -| `HEAD /api/:version/projects/:id/repository/files/:file_path/raw` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | -| `POST /api/:version/internal/allowed` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | -| `POST /api/:version/internal/lfs_authenticate` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | -| `POST /api/:version/internal/post_receive` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | -| `POST /api/:version/internal/pre_receive` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | -| `POST /api/:version/projects/:id/approvals` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/project_approvals.rb) | -| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/approvals` | `:low` | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | -| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/approve` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | -| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/unapprove` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb)| -| `POST /api/:version/projects/:id/protected_branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb)| -| `POST /api/:version/projects/:id/repository/commits` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| -| `POST /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | -| `PUT /api/:version/projects/:id/push_rule` | default | | -| `PUT /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | -| `Projects::BlameController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blame_controller.rb) | -| `Projects::BlobController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | -| `Projects::BlobController#diff` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | -| `Projects::BlobController#edit` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | -| `Projects::BlobController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | -| `Projects::BlobController#update` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | -| `Projects::BranchesController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | -| `Projects::BranchesController#destroy` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | -| `Projects::BranchesController#diverging_commit_counts` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | -| `Projects::BranchesController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | -| `Projects::BranchesController#new` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | -| `Projects::CommitController#branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | -| `Projects::CommitController#merge_requests` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | -| `Projects::CommitController#pipelines` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | -| `Projects::CommitController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | -| `Projects::CommitsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb)| -| `Projects::CommitsController#signatures` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb) | -| `Projects::CompareController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb) | -| `Projects::CompareController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | -| `Projects::CompareController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | -| `Projects::CompareController#signatures` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | -| `Projects::FindFileController#list` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/find_file_controller.rb) | -| `Projects::FindFileController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/find_file_controller.rb) | -| `Projects::ForksController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/forks_controller.rb) | -| `Projects::GraphsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/graphs_controller.rb) | -| `Projects::NetworkController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/network_controller.rb) | -| `Projects::PathLocksController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/controllers/projects/path_locks_controller.rb) | -| `Projects::RawController#show` | default | | -| `Projects::RefsController#logs_tree` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/refs_controller.rb) | -| `Projects::RefsController#switch` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/refs_controller.rb) | -| `Projects::RepositoriesController#archive` | default | | -| `Projects::Settings::RepositoryController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/settings/repository_controller.rb) | -| `Projects::TagsController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | -| `Projects::TagsController#new` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | -| `Projects::TagsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | -| `Projects::TemplatesController#names` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/templates_controller.rb) | -| `Projects::TreeController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tree_controller.rb) | -| `ProjectsController#refs` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects_controller.rb) | -| `Repositories::GitHttpController#git_receive_pack` | default | | -| `Repositories::GitHttpController#git_upload_pack` | default | | -| `Repositories::GitHttpController#info_refs` | default | | -| `Repositories::LfsApiController#batch` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_api_controller.rb) | -| `Repositories::LfsLocksApiController#verify` | default | | -| `Repositories::LfsStorageController#download` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | -| `Repositories::LfsStorageController#upload_authorize` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | -| `Repositories::LfsStorageController#upload_finalize` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | +Gitaly is a Golang RPC service which handles all the `git` calls made by GitLab. +GitLab is not exposed directly, and all traffic comes through Create: Source Code. +For more information, read [Gitaly touch points](gitaly_touch_points.md). + +### Source Code REST API Endpoints + +Create: Source Code has over 100 REST endpoints, being a mixture of Grape API endpoints and Rails controller endpoints. +For a detailed list, refer to [Source Code REST Endpoints](rest_endpoints.md). diff --git a/doc/development/backend/create_source_code_be/rest_endpoints.md b/doc/development/backend/create_source_code_be/rest_endpoints.md new file mode 100644 index 00000000000..dd43bb914c9 --- /dev/null +++ b/doc/development/backend/create_source_code_be/rest_endpoints.md @@ -0,0 +1,112 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Source Code REST endpoints + +The Create :: Source Code team maintains these endpoints: + +| Endpoint | Threshold | Source | +| -----------------------------------------------------------------------------------|---------------------------------------|--------------------------------------------------------------------------------------| +| `DELETE /api/:version/projects/:id/protected_branches/:name` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | +| `GET /api/:version/internal/authorized_keys` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | | | +| `GET /api/:version/internal/lfs` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/lfs.rb)| +| `GET /api/:version/projects/:id/approval_rules` | `:low` | | +| `GET /api/:version/projects/:id/approval_settings` | default | | +| `GET /api/:version/projects/:id/approvals` | default | | +| `GET /api/:version/projects/:id/forks` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | +| `GET /api/:version/projects/:id/groups` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | +| `GET /api/:version/projects/:id/languages` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/projects.rb) | +| `GET /api/:version/projects/:id/merge_request_approval_setting` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/merge_request_approval_settings.rb) | +| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_rules` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/merge_request_approval_rules.rb) | +| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_settings` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/project_approval_settings.rb) | +| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approval_state` | `:low` | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | +| `GET /api/:version/projects/:id/merge_requests/:merge_request_iid/approvals` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | +| `GET /api/:version/projects/:id/protected_branches` | default |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | +| `GET /api/:version/projects/:id/protected_branches/:name` | default |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb) | +| `GET /api/:version/projects/:id/protected_tags` | default | | +| `GET /api/:version/projects/:id/protected_tags/:name` | default | | +| `GET /api/:version/projects/:id/push_rule` | default | | +| `GET /api/:version/projects/:id/remote_mirrors` | default | | +| `GET /api/:version/projects/:id/repository/archive` | default | | +| `GET /api/:version/projects/:id/repository/blobs/:sha` | default | | +| `GET /api/:version/projects/:id/repository/blobs/:sha/raw` | default | | +| `GET /api/:version/projects/:id/repository/branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/branches.rb) | +| `GET /api/:version/projects/:id/repository/branches/:branch` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/branches.rb) | +| `GET /api/:version/projects/:id/repository/commits` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| +| `GET /api/:version/projects/:id/repository/commits/:sha` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | +| `GET /api/:version/projects/:id/repository/commits/:sha/comments` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | +| `GET /api/:version/projects/:id/repository/commits/:sha/diff` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | +| `GET /api/:version/projects/:id/repository/commits/:sha/merge_requests` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| +| `GET /api/:version/projects/:id/repository/commits/:sha/refs` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb) | +| `GET /api/:version/projects/:id/repository/compare` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/repositories.rb) | +| `GET /api/:version/projects/:id/repository/contributors` | default | | +| `GET /api/:version/projects/:id/repository/files/:file_path` | default | | +| `GET /api/:version/projects/:id/repository/files/:file_path/raw` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | +| `GET /api/:version/projects/:id/repository/tags` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/tags.rb) | +| `GET /api/:version/projects/:id/repository/tree` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/repositories.rb) | +| `GET /api/:version/projects/:id/statistics` | default | | +| `GraphqlController#execute` | default | | +| `HEAD /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | +| `HEAD /api/:version/projects/:id/repository/files/:file_path/raw` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | +| `POST /api/:version/internal/allowed` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | +| `POST /api/:version/internal/lfs_authenticate` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | +| `POST /api/:version/internal/post_receive` | default | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | +| `POST /api/:version/internal/pre_receive` | `:high` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/internal/base.rb) | +| `POST /api/:version/projects/:id/approvals` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/project_approvals.rb) | +| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/approvals` | `:low` | [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | +| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/approve` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb) | +| `POST /api/:version/projects/:id/merge_requests/:merge_request_iid/unapprove` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/merge_request_approvals.rb)| +| `POST /api/:version/projects/:id/protected_branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/protected_branches.rb)| +| `POST /api/:version/projects/:id/repository/commits` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/commits.rb)| +| `POST /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | +| `PUT /api/:version/projects/:id/push_rule` | default | | +| `PUT /api/:version/projects/:id/repository/files/:file_path` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/files.rb) | +| `Projects::BlameController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blame_controller.rb) | +| `Projects::BlobController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | +| `Projects::BlobController#diff` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | +| `Projects::BlobController#edit` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | +| `Projects::BlobController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | +| `Projects::BlobController#update` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/blob_controller.rb) | +| `Projects::BranchesController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | +| `Projects::BranchesController#destroy` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | +| `Projects::BranchesController#diverging_commit_counts` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | +| `Projects::BranchesController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | +| `Projects::BranchesController#new` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/branches_controller.rb) | +| `Projects::CommitController#branches` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | +| `Projects::CommitController#merge_requests` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | +| `Projects::CommitController#pipelines` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | +| `Projects::CommitController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commit_controller.rb) | +| `Projects::CommitsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb)| +| `Projects::CommitsController#signatures` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb) | +| `Projects::CompareController#create` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/commits_controller.rb) | +| `Projects::CompareController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | +| `Projects::CompareController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | +| `Projects::CompareController#signatures` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/compare_controller.rb) | +| `Projects::FindFileController#list` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/find_file_controller.rb) | +| `Projects::FindFileController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/find_file_controller.rb) | +| `Projects::ForksController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/forks_controller.rb) | +| `Projects::GraphsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/graphs_controller.rb) | +| `Projects::NetworkController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/network_controller.rb) | +| `Projects::PathLocksController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/controllers/projects/path_locks_controller.rb) | +| `Projects::RawController#show` | default | | +| `Projects::RefsController#logs_tree` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/refs_controller.rb) | +| `Projects::RefsController#switch` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/refs_controller.rb) | +| `Projects::RepositoriesController#archive` | default | | +| `Projects::Settings::RepositoryController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/settings/repository_controller.rb) | +| `Projects::TagsController#index` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | +| `Projects::TagsController#new` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | +| `Projects::TagsController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tags_controller.rb) | +| `Projects::TemplatesController#names` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/templates_controller.rb) | +| `Projects::TreeController#show` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects/tree_controller.rb) | +| `ProjectsController#refs` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/projects_controller.rb) | +| `Repositories::GitHttpController#git_receive_pack` | default | | +| `Repositories::GitHttpController#git_upload_pack` | default | | +| `Repositories::GitHttpController#info_refs` | default | | +| `Repositories::LfsApiController#batch` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_api_controller.rb) | +| `Repositories::LfsLocksApiController#verify` | default | | +| `Repositories::LfsStorageController#download` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | +| `Repositories::LfsStorageController#upload_authorize` | `:medium` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | +| `Repositories::LfsStorageController#upload_finalize` | `:low` |[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/controllers/repositories/lfs_storage_controller.rb) | diff --git a/doc/development/backend/ruby_style_guide.md b/doc/development/backend/ruby_style_guide.md index 419db628b0d..6c8125a6157 100644 --- a/doc/development/backend/ruby_style_guide.md +++ b/doc/development/backend/ruby_style_guide.md @@ -13,7 +13,7 @@ Generally, if a style is not covered by [existing Rubocop rules or style guides] Before adding a new cop to enforce a given style, make sure to discuss it with your team. When the style is approved by a backend EM or by a BE staff eng, add a new section to this page to document the new rule. For every new guideline, add it in a new section and link the discussion from the section's -[version history note](../documentation/styleguide/index.md#version-text-in-the-version-history) +[version history note](../documentation/versions.md#add-a-version-history-item) to provide context and serve as a reference. Just because something is listed here does not mean it cannot be reopened for discussion. diff --git a/doc/development/batched_background_migrations.md b/doc/development/batched_background_migrations.md index e7703b5dd2b..f5f3655944b 100644 --- a/doc/development/batched_background_migrations.md +++ b/doc/development/batched_background_migrations.md @@ -1,319 +1,11 @@ --- -type: reference, dev -stage: Enablement -group: Database -info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +redirect_to: 'database/batched_background_migrations.md' +remove_date: '2022-07-26' --- -# Batched background migrations +This document was moved to [another location](database/batched_background_migrations.md). -Batched Background Migrations should be used to perform data migrations whenever a -migration exceeds [the time limits](migration_style_guide.md#how-long-a-migration-should-take) -in our guidelines. For example, you can use batched background -migrations to migrate data that's stored in a single JSON column -to a separate table instead. - -## When to use batched background migrations - -Use a batched background migration when you migrate _data_ in tables containing -so many rows that the process would exceed -[the time limits in our guidelines](migration_style_guide.md#how-long-a-migration-should-take) -if performed using a regular Rails migration. - -- Batched background migrations should be used when migrating data in - [high-traffic tables](migration_style_guide.md#high-traffic-tables). -- Batched background migrations may also be used when executing numerous single-row queries - for every item on a large dataset. Typically, for single-record patterns, runtime is - largely dependent on the size of the dataset. Split the dataset accordingly, - and put it into background migrations. -- Don't use batched background migrations to perform schema migrations. - -Background migrations can help when: - -- Migrating events from one table to multiple separate tables. -- Populating one column based on JSON stored in another column. -- Migrating data that depends on the output of external services. (For example, an API.) - -NOTE: -If the batched background migration is part of an important upgrade, it must be announced -in the release post. Discuss with your Project Manager if you're unsure if the migration falls -into this category. - -## Isolation - -Batched background migrations must be isolated and can not use application code. (For example, -models defined in `app/models`.). Because these migrations can take a long time to -run, it's possible for new versions to deploy while the migrations are still running. - -## Idempotence - -Batched background migrations are executed in a context of a Sidekiq process. -The usual Sidekiq rules apply, especially the rule that jobs should be small -and idempotent. Make sure that in case that your migration job is retried, data -integrity is guaranteed. - -See [Sidekiq best practices guidelines](https://github.com/mperham/sidekiq/wiki/Best-Practices) -for more details. - -## Batched background migrations for EE-only features - -All the background migration classes for EE-only features should be present in GitLab CE. -For this purpose, create an empty class for GitLab CE, and extend it for GitLab EE -as explained in the guidelines for -[implementing Enterprise Edition features](ee_features.md#code-in-libgitlabbackground_migration). - -Batched Background migrations are simple classes that define a `perform` method. A -Sidekiq worker then executes such a class, passing any arguments to it. All -migration classes must be defined in the namespace -`Gitlab::BackgroundMigration`. Place the files in the directory -`lib/gitlab/background_migration/`. - -## Queueing - -Queueing a batched background migration should be done in a post-deployment -migration. Use this `queue_batched_background_migration` example, queueing the -migration to be executed in batches. Replace the class name and arguments with the values -from your migration: - -```ruby -queue_batched_background_migration( - JOB_CLASS_NAME, - TABLE_NAME, - JOB_ARGUMENTS, - JOB_INTERVAL - ) -``` - -Make sure the newly-created data is either migrated, or -saved in both the old and new version upon creation. Removals in -turn can be handled by defining foreign keys with cascading deletes. - -### Requeuing batched background migrations - -If one of the batched background migrations contains a bug that is fixed in a patch -release, you must requeue the batched background migration so the migration -repeats on systems that already performed the initial migration. - -When you requeue the batched background migration, turn the original -queuing into a no-op by clearing up the `#up` and `#down` methods of the -migration performing the requeuing. Otherwise, the batched background migration is -queued multiple times on systems that are upgrading multiple patch releases at -once. - -When you start the second post-deployment migration, delete the -previously batched migration with the provided code: - -```ruby -Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION_NAME, TABLE_NAME, COLUMN, JOB_ARGUMENTS) - .delete_all -``` - -## Cleaning up - -NOTE: -Cleaning up any remaining background migrations must be done in either a major -or minor release. You must not do this in a patch release. - -Because background migrations can take a long time, you can't immediately clean -things up after queueing them. For example, you can't drop a column used in the -migration process, as jobs would fail. You must add a separate _post-deployment_ -migration in a future release that finishes any remaining -jobs before cleaning things up. (For example, removing a column.) - -To migrate the data from column `foo` (containing a big JSON blob) to column `bar` -(containing a string), you would: - -1. Release A: - 1. Create a migration class that performs the migration for a row with a given ID. - 1. Update new rows using one of these techniques: - - Create a new trigger for simple copy operations that don't need application logic. - - Handle this operation in the model/service as the records are created or updated. - - Create a new custom background job that updates the records. - 1. Queue the batched background migration for all existing rows in a post-deployment migration. -1. Release B: - 1. Add a post-deployment migration that checks if the batched background migration is completed. - 1. Deploy code so that the application starts using the new column and stops to update new records. - 1. Remove the old column. - -Bump to the [import/export version](../user/project/settings/import_export.md) may -be required, if importing a project from a prior version of GitLab requires the -data to be in the new format. - -## Example - -The table `integrations` has a field called `properties`, stored in JSON. For all rows, -extract the `url` key from this JSON object and store it in the `integrations.url` -column. Millions of integrations exist, and parsing JSON is slow, so you can't -do this work in a regular migration. - -1. Start by defining our migration class: - - ```ruby - class Gitlab::BackgroundMigration::ExtractIntegrationsUrl - class Integration < ActiveRecord::Base - self.table_name = 'integrations' - end - - def perform(start_id, end_id) - Integration.where(id: start_id..end_id).each do |integration| - json = JSON.load(integration.properties) - - integration.update(url: json['url']) if json['url'] - rescue JSON::ParserError - # If the JSON is invalid we don't want to keep the job around forever, - # instead we'll just leave the "url" field to whatever the default value - # is. - next - end - end - end - ``` - - NOTE: - To get a `connection` in the batched background migration,use an inheritance - relation using the following base class `Gitlab::BackgroundMigration::BaseJob`. - For example: `class Gitlab::BackgroundMigration::ExtractIntegrationsUrl < Gitlab::BackgroundMigration::BaseJob` - -1. Add a new trigger to the database to update newly created and updated integrations, - similar to this example: - - ```ruby - execute(<<~SQL) - CREATE OR REPLACE FUNCTION example() RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - NEW."url" := NEW.properties -> "url" - RETURN NEW; - END; - $$; - SQL - ``` - -1. Create a post-deployment migration that queues the migration for existing data: - - ```ruby - class QueueExtractIntegrationsUrl < Gitlab::Database::Migration[1.0] - disable_ddl_transaction! - - MIGRATION = 'ExtractIntegrationsUrl' - DELAY_INTERVAL = 2.minutes - - def up - queue_batched_background_migration( - MIGRATION, - :migrations, - :id, - job_interval: DELAY_INTERVAL - ) - end - - def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :migrations, :id, []).delete_all - end - end - ``` - - After deployment, our application: - - Continues using the data as before. - - Ensures that both existing and new data are migrated. - -1. In the next release, remove the trigger. We must also add a new post-deployment migration - that checks that the batched background migration is completed. For example: - - ```ruby - class FinalizeExtractIntegrationsUrlJobs < Gitlab::Database::Migration[1.0] - MIGRATION = 'ExtractIntegrationsUrl' - disable_ddl_transaction! - - def up - ensure_batched_background_migration_is_finished( - job_class_name: MIGRATION, - table_name: :integrations, - column_name: :id, - job_arguments: [] - ) - end - - def down - # no-op - end - end - ``` - - If the application does not depend on the data being 100% migrated (for - instance, the data is advisory, and not mission-critical), then you can skip this - final step. This step confirms that the migration is completed, and all of the rows were migrated. - -After the batched migration is completed, you can safely remove the `integrations.properties` column. - -## Testing - -Writing tests is required for: - -- The batched background migrations' queueing migration. -- The batched background migration itself. -- A cleanup migration. - -The `:migration` and `schema: :latest` RSpec tags are automatically set for -background migration specs. Refer to the -[Testing Rails migrations](testing_guide/testing_migrations_guide.md#testing-a-non-activerecordmigration-class) -style guide. - -Remember that `before` and `after` RSpec hooks -migrate your database down and up. These hooks can result in other batched background -migrations being called. Using `spy` test doubles with -`have_received` is encouraged, instead of using regular test doubles, because -your expectations defined in a `it` block can conflict with what is -called in RSpec hooks. Refer to [issue #35351](https://gitlab.com/gitlab-org/gitlab/-/issues/18839) -for more details. - -## Best practices - -1. Know how much data you're dealing with. -1. Make sure the batched background migration jobs are idempotent. -1. Confirm the tests you write are not false positives. -1. If the data being migrated is critical and cannot be lost, the - clean-up migration must also check the final state of the data before completing. -1. Discuss the numbers with a database specialist. The migration may add - more pressure on DB than you expect. Measure on staging, - or ask someone to measure on production. -1. Know how much time is required to run the batched background migration. - -## Additional tips and strategies - -### Viewing failure error logs - -You can view failures in two ways: - -- Via GitLab logs: - 1. After running a batched background migration, if any jobs fail, - view the logs in [Kibana](https://log.gprd.gitlab.net/goto/5f06a57f768c6025e1c65aefb4075694). - View the production Sidekiq log and filter for: - - - `json.new_state: failed` - - `json.job_class_name: <Batched Background Migration job class name>` - - `json.job_arguments: <Batched Background Migration job class arguments>` - - 1. Review the `json.exception_class` and `json.exception_message` values to help - understand why the jobs failed. - - 1. Remember the retry mechanism. Having a failure does not mean the job failed. - Always check the last status of the job. - -- Via database: - - 1. Get the batched background migration `CLASS_NAME`. - 1. Execute the following query in the PostgreSQL console: - - ```sql - SELECT migration.id, migration.job_class_name, transition_logs.exception_class, transition_logs.exception_message - FROM batched_background_migrations as migration - INNER JOIN batched_background_migration_jobs as jobs - ON jobs.batched_background_migration_id = migration.id - INNER JOIN batched_background_migration_job_transition_logs as transition_logs - ON transition_logs.batched_background_migration_job_id = jobs.id - WHERE transition_logs.next_status = '2' AND migration.job_class_name = "CLASS_NAME"; - ``` +<!-- This redirect file can be deleted after <2022-07-26>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/changelog.md b/doc/development/changelog.md index b98ed6cb109..c19c5b40382 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -100,7 +100,7 @@ EE: true database records created during Cycle Analytics model spec." - _Any_ contribution from a community member, no matter how small, **may** have a changelog entry regardless of these guidelines if the contributor wants one. -- Any [GLEX experiment](experiment_guide/gitlab_experiment.md) changes **should not** have a changelog entry. +- Any [experiment](experiment_guide/index.md) changes **should not** have a changelog entry. - An MR that includes only documentation changes **should not** have a changelog entry. For more information, see diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index e18fcb0061b..2065021c61b 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -20,12 +20,10 @@ tasks such as: To request access to ChatOps on GitLab.com: 1. Sign in to [Internal GitLab for Operations](https://ops.gitlab.net/users/sign_in) - with one of the following methods: + with one of the following methods (Okta is not supported): - - The same username you use on GitLab.com. You may have to choose a different - username later. + - The same username you use on GitLab.com. You may have to choose a different username later. - Clicking the **Sign in with Google** button to sign in with your GitLab.com email address. - - Clicking the **Sign in with Okta** button to sign in with Okta. 1. Confirm that your username in [Internal GitLab for Operations](https://ops.gitlab.net/) is the same as your username in [GitLab.com](https://gitlab.com/). If the usernames diff --git a/doc/development/cicd/cicd_reference_documentation_guide.md b/doc/development/cicd/cicd_reference_documentation_guide.md index e937220d208..0da1717f53c 100644 --- a/doc/development/cicd/cicd_reference_documentation_guide.md +++ b/doc/development/cicd/cicd_reference_documentation_guide.md @@ -4,7 +4,7 @@ group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# CI/CD YAML reference style guide **(FREE)** +# Documenting the `.gitlab-ci.yml` keywords **(FREE)** The [CI/CD YAML reference](../../ci/yaml/index.md) uses a standard style to make it easier to use and update. diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index c6f59a7e452..8d88e7155a2 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -342,32 +342,6 @@ include: - remote: https://gitlab.com/gitlab-org/gitlab/-/raw/v13.0.1-ee/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml ``` -### Use a feature flag to roll out a `latest` template - -With a major version release like 13.0 or 14.0, [stable templates](#stable-version) must be -updated with their corresponding [latest template versions](#latest-version). -It may be hard to gauge the impact of this change, so use the `redirect_to_latest_template_<name>` -feature flag to test the impact on a subset of users. Using a feature flag can help -reduce the risk of reverts or rollbacks on production. - -For example, to redirect the stable `Jobs/Deploy` template to its latest template in 25% of -projects on `gitlab.com`: - -```shell -/chatops run feature set redirect_to_latest_template_jobs_deploy 25 --actors -``` - -After you're confident the latest template can be moved to stable: - -1. Update the stable template with the content of the latest version. -1. Remove the migration template from `Gitlab::Template::GitlabCiYmlTemplate::TEMPLATES_WITH_LATEST_VERSION` const. -1. Remove the corresponding feature flag. - -NOTE: -Feature flags are enabled by default in RSpec, so all tests are performed -against the latest templates. You should also test the stable templates -with `stub_feature_flags(redirect_to_latest_template_<name>: false)`. - ### Further reading There is an [open issue](https://gitlab.com/gitlab-org/gitlab/-/issues/17716) about diff --git a/doc/development/code_intelligence/index.md b/doc/development/code_intelligence/index.md index e1e2105298c..3a8845084c3 100644 --- a/doc/development/code_intelligence/index.md +++ b/doc/development/code_intelligence/index.md @@ -38,7 +38,7 @@ sequenceDiagram 1. The CI/CD job generates a document in an LSIF format (usually `dump.lsif`) using [an indexer](https://lsif.dev) for the language of a project. The format [describes](https://github.com/sourcegraph/sourcegraph/blob/main/doc/code_intelligence/explanations/writing_an_indexer.md) - interactions between a method or function and its definition(s) or references. The + interactions between a method or function and its definitions or references. The document is marked to be stored as an LSIF report artifact. 1. After receiving a request for storing the artifact, Workhorse asks diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 48bbe4c60ba..252bd1daf55 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -26,7 +26,7 @@ This is only a recommendation and the reviewer may be from a different team. However, it is recommended to pick someone who is a [domain expert](#domain-experts). If your merge request touches more than one domain (for example, Dynamic Analysis and GraphQL), ask for reviews from an expert from each domain. -You can read more about the importance of involving reviewer(s) in the section on the responsibility of the author below. +You can read more about the importance of involving reviewers in the section on the responsibility of the author below. If you need some guidance (for example, it's your first merge request), feel free to ask one of the [Merge request coaches](https://about.gitlab.com/company/team/). @@ -107,7 +107,7 @@ For more information, review [the roulette README](https://gitlab.com/gitlab-org ### Approval guidelines As described in the section on the responsibility of the maintainer below, you -are recommended to get your merge request approved and merged by maintainer(s) +are recommended to get your merge request approved and merged by maintainers with [domain expertise](#domain-experts). 1. If your merge request includes backend changes (*1*), it must be @@ -118,8 +118,7 @@ with [domain expertise](#domain-experts). 1. If your merge request includes frontend changes (*1*), it must be **approved by a [frontend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_frontend)**. 1. If your merge request includes user-facing changes (*3*), it must be - **approved by a [Product Designer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_UX)**, - based on assignments in the appropriate [DevOps stage group](https://about.gitlab.com/handbook/product/categories/#devops-stages). + **approved by a [Product Designer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_UX)**. See the [design and user interface guidelines](contributing/design.md) for details. 1. If your merge request includes adding a new JavaScript library (*1*)... - If the library significantly increases the @@ -155,7 +154,7 @@ with [domain expertise](#domain-experts). #### Acceptance checklist -This checklist encourages the authors, reviewers, and maintainers of merge requests (MRs) to confirm changes were analyzed for high-impact risks to quality, performance, reliability, security, and maintainability. +This checklist encourages the authors, reviewers, and maintainers of merge requests (MRs) to confirm changes were analyzed for high-impact risks to quality, performance, reliability, security, observability, and maintainability. Using checklists improves quality in software engineering. This checklist is a straightforward tool to support and bolster the skills of contributors to the GitLab codebase. @@ -183,6 +182,10 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering 1. I have considered the scalability risk based on future predicted growth. 1. I have considered the performance, reliability, and availability impacts of this change on large customers who may have significantly more data than the average customer. +##### Observability instrumentation + +1. I have included enough instrumentation to facilitate debugging and proactive performance improvements through observability. + ##### Documentation 1. I have included changelog trailers, or I have decided that they are not needed. @@ -220,7 +223,9 @@ should be confident that: The best way to do this, and to avoid unnecessary back-and-forth with reviewers, is to perform a self-review of your own merge request, following the -[Code Review](#reviewing-a-merge-request) guidelines. +[Code Review](#reviewing-a-merge-request) guidelines. During this self-review, +try to include comments in the MR on lines +where decisions or trade-offs were made, or where a contextual explanation might aid the reviewer in more easily understanding the code. To reach the required level of confidence in their solution, an author is expected to involve other people in the investigation and implementation processes as @@ -258,7 +263,7 @@ Avoid: [_explain why, not what_](https://blog.codinghorror.com/code-tells-you-how-comments-tell-you-why/). - Requesting maintainer reviews of merge requests with failed tests. If the tests are failing and you have to request a review, ensure you leave a comment with an explanation. - Excessively mentioning maintainers through email or Slack (if the maintainer is reachable -through Slack). If you can't add a reviewer for a merge request, it's acceptable to `@` mention a maintainer in a comment. In all other cases, it's sufficient to add a reviewer or [request their attention](../user/project/merge_requests/index.md#request-attention-to-a-merge-request) if they're already a reviewer. +through Slack). If you can't add a reviewer for a merge request, `@` mentioning a maintainer in a comment is acceptable and in all other cases adding a reviewer is sufficient. This saves reviewers time and helps authors catch mistakes earlier. @@ -268,8 +273,10 @@ This saves reviewers time and helps authors catch mistakes earlier. that it meets all requirements, you should: - Click the Approve button. -- Request a review from a maintainer or [request their attention](../user/project/merge_requests/index.md#request-attention-to-a-merge-request) if they're already a reviewer. Default to requests for a maintainer with [domain expertise](#domain-experts), +- `@` mention the author to generate a to-do notification, and advise them that their merge request has been reviewed and approved. +- Request a review from a maintainer. Default to requests for a maintainer with [domain expertise](#domain-experts), however, if one isn't available or you think the merge request doesn't need a review by a [domain expert](#domain-experts), feel free to follow the [Reviewer roulette](#reviewer-roulette) suggestion. +- Remove yourself as a reviewer. ### The responsibility of the maintainer @@ -297,7 +304,7 @@ If a developer who happens to also be a maintainer was involved in a merge reque as a reviewer, it is recommended that they are not also picked as the maintainer to ultimately approve and merge it. Maintainers should check before merging if the merge request is approved by the -required approvers. If still awaiting further approvals from others, explain that in a comment and [request attention](../user/project/merge_requests/index.md#request-attention-to-a-merge-request) from other reviewers as appropriate. Do not remove yourself as a reviewer. +required approvers. If still awaiting further approvals from others, remove yourself as a reviewer then `@` mention the author and explain why in a comment. Stay as reviewer if you're merging the code. Maintainers must check before merging if the merge request is introducing new vulnerabilities, by inspecting the list in the merge request @@ -319,20 +326,14 @@ After merging, a maintainer should stay as the reviewer listed on the merge requ ### Dogfooding the Reviewers feature -Replaced with [dogfooding the attention request feature](#dogfooding-the-attention-request-feature). - -### Dogfooding the attention request feature - -In March of 2022, an updated process was put in place aimed at efficiently and consistently dogfooding the -[attention requests feature](../user/project/merge_requests/index.md#request-attention-to-a-merge-request) under `Merge requests` -> `Need your attention`. This replaces previous guidance on [dogfooding the reviewers feature](#dogfooding-the-reviewers-feature). +On March 18th 2021, an updated process was put in place aimed at efficiently and consistently dogfooding the Reviewers feature. Here is a summary of the changes, also reflected in this section above. -- Merge request authors and DRIs stay as assignees -- Assignees request a review from reviewer(s) when they are expected to review -- Reviewers stay assigned for the entire duration of the merge request -- Reviewers request attention from the assignee or other reviewer(s) after they've done reviewing, depending on who needs to take action -- Assignees request attention from the reviewer(s) when changes are made +- Merge request authors and DRIs stay as Assignees +- Authors request a review from Reviewers when they are expected to review +- Reviewers remove themselves after they're done reviewing/approving +- The last approver stays as Reviewer upon merging ## Best practices @@ -443,7 +444,7 @@ experience, refactors the existing code). Then: - For non-mandatory suggestions, decorate with (non-blocking) so the author knows they can optionally resolve within the merge request or follow-up at a later stage. - There's a [Chrome/Firefox add-on](https://gitlab.com/conventionalcomments/conventional-comments-button) which you can use to apply [Conventional Comment](https://conventionalcomments.org/) prefixes. -- Ensure there are no open dependencies. Check [linked issues](../user/project/issues/related_issues.md) for blockers. Clarify with the author(s) +- Ensure there are no open dependencies. Check [linked issues](../user/project/issues/related_issues.md) for blockers. Clarify with the authors if necessary. If blocked by one or more open MRs, set an [MR dependency](../user/project/merge_requests/merge_request_dependencies.md). - After a round of line notes, it can be helpful to post a summary note such as "Looks good to me", or "Just a couple things to address." @@ -696,10 +697,10 @@ Properties of customer critical merge requests: - The [VP of Development](https://about.gitlab.com/job-families/engineering/development/management/vp/) ([@clefelhocz1](https://gitlab.com/clefelhocz1)) is the DRI for deciding if a merge request qualifies as customer critical. - The DRI applies the `customer-critical-merge-request` label to the merge request. -- It is required that the reviewer(s) and maintainer(s) involved with a customer critical merge request are engaged as soon as this decision is made. +- It is required that the reviewers and maintainers involved with a customer critical merge request are engaged as soon as this decision is made. - It is required to prioritize work for those involved on a customer critical merge request so that they have the time available necessary to focus on it. - It is required to adhere to GitLab [values](https://about.gitlab.com/handbook/values/) and processes when working on customer critical merge requests, taking particular note of family and friends first/work second, definition of done, iteration, and release when it's ready. -- Customer critical merge requests are required to not reduce security, introduce data-loss risk, reduce availability, nor break existing functionality per the process for [prioritizing technical decisions](https://about.gitlab.com/handbook/engineering/principles/#prioritizing-technical-decisions). +- Customer critical merge requests are required to not reduce security, introduce data-loss risk, reduce availability, nor break existing functionality per the process for [prioritizing technical decisions](https://about.gitlab.com/handbook/engineering/development/principles/#prioritizing-technical-decisions). - On customer critical requests, it is _recommended_ that those involved _consider_ coordinating synchronously (Zoom, Slack) in addition to asynchronously (merge requests comments) if they believe this may reduce the elapsed time to merge even though this _may_ sacrifice [efficiency](https://about.gitlab.com/company/culture/all-remote/asynchronous/#evaluating-efficiency.md). - After a customer critical merge request is merged, a retrospective must be completed with the intention of reducing the frequency of future customer critical merge requests. diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index def39a960d8..7f5c800216a 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -117,7 +117,7 @@ At any moment, but usually _during_ or _after_ the design's implementation: for additions or enhancements to the design system. - Create issues with the [`~UX debt`](issue_workflow.md#technical-and-ux-debt) label for intentional deviations from the agreed-upon UX requirements due to - time or feasibility challenges, linking back to the corresponding issue(s) or - MR(s). + time or feasibility challenges, linking back to the corresponding issues or + merge requests. - Create issues for [feature additions or enhancements](issue_workflow.md#feature-proposals) outside the agreed-upon UX requirements to avoid scope creep. diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index ea54f36a7e5..8a4b06840a4 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -33,9 +33,8 @@ GitLab Inc engineers should refer to the [engineering workflow document](https:/ ## Security vulnerability disclosure -Report suspected security vulnerabilities in private to -`support@gitlab.com`, also see the -[disclosure section on the GitLab.com website](https://about.gitlab.com/security/disclosure/). +Report suspected security vulnerabilities by following the +[disclosure process on the GitLab.com website](https://about.gitlab.com/security/disclosure/). WARNING: Do **NOT** create publicly viewable issues for suspected security vulnerabilities. diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index fe1549e7f34..97c8c179e09 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -108,7 +108,7 @@ Group labels specify which [groups](https://about.gitlab.com/company/team/struct It's highly recommended to add a group label, as it's used by our triage automation to -[infer the correct stage label](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues). +[infer the correct stage label](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues-and-merge-requests). #### Naming and color convention diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 5ed0885eed9..ee1ed744cd4 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -53,7 +53,7 @@ request is as follows: 1. If you have multiple commits, combine them into a few logically organized commits by [squashing them](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#_squashing), but do not change the commit history if you're working on shared branches though. -1. Push the commit(s) to your working branch in your fork. +1. Push the commits to your working branch in your fork. 1. Submit a merge request (MR) to the `main` branch in the main GitLab project. 1. Your merge request needs at least 1 approval, but depending on your changes you might need additional approvals. Refer to the [Approval guidelines](../code_review.md#approval-guidelines). @@ -65,7 +65,7 @@ request is as follows: template already provided in the "Description" field. 1. If you are contributing documentation, choose `Documentation` from the "Choose a template" menu and fill in the description according to the template. - 1. Use the syntax `Solves #XXX`, `Closes #XXX`, or `Refs #XXX` to mention the issue(s) your merge + 1. Use the syntax `Solves #XXX`, `Closes #XXX`, or `Refs #XXX` to mention the issues your merge request addresses. Referenced issues do not [close automatically](../../user/project/issues/managing_issues.md#closing-issues-automatically). You must close them manually once the merge request is merged. 1. The MR must include *Before* and *After* screenshots if UI changes are made. @@ -81,6 +81,7 @@ request is as follows: 1. If your MR touches code that executes shell commands, reads or opens files, or handles paths to files on disk, make sure it adheres to the [shell command guidelines](../shell_commands.md) +1. [Code changes should include observability instrumentation](../code_review.md#observability-instrumentation). 1. If your code needs to handle file storage, see the [uploads documentation](../uploads/index.md). 1. If your merge request adds one or more migrations, make sure to execute all migrations on a fresh database before the MR is reviewed. If the review leads diff --git a/doc/development/contributing/verify/index.md b/doc/development/contributing/verify/index.md index 828eb0a9598..01aacffd00f 100644 --- a/doc/development/contributing/verify/index.md +++ b/doc/development/contributing/verify/index.md @@ -231,5 +231,5 @@ building medical, aviation, and automotive software. Continuous Integration is a mission critical part of software engineering. When you are working on a subsystem for pipeline processing and transitioning -CI/CD statuses, request an additional review from a domain expert and hold -others accountable for doing the same. +CI/CD statuses, request an additional opinion on the design from a domain expert +as early as possible and hold others accountable for doing the same. diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index f941e0720c6..003df4fe078 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -66,7 +66,7 @@ continue to apply. However, there are a few things that deserve special emphasis Danger is a powerful tool and flexible tool, but not always the most appropriate way to solve a given problem or workflow. -First, be aware of the GitLab [commitment to dogfooding](https://about.gitlab.com/handbook/engineering/principles/#dogfooding). +First, be aware of the GitLab [commitment to dogfooding](https://about.gitlab.com/handbook/engineering/development/principles/#dogfooding). The code we write for Danger is GitLab-specific, and it **may not** be most appropriate place to implement functionality that addresses a need we encounter. Our users, customers, and even our own satellite projects, such as [Gitaly](https://gitlab.com/gitlab-org/gitaly), @@ -155,7 +155,7 @@ To enable the Dangerfile on another existing GitLab project, complete the follow file: - '/ci/danger-review.yml' rules: - - if: '$CI_SERVER_HOST == "gitlab.com"' + - if: $CI_SERVER_HOST == "gitlab.com" ``` 1. If your project is in the `gitlab-org` group, you don't need to set up any token as the `DANGER_GITLAB_API_TOKEN` @@ -196,10 +196,11 @@ is not shared to forks. Contributors can configure Danger for their forks with the following steps: -1. Add an [environment variable](../ci/variables/index.md) called `DANGER_GITLAB_API_TOKEN` with a -[personal API token](https://gitlab.com/-/profile/personal_access_tokens?name=GitLab+Dangerbot&scopes=api) -to your fork that has the `api` scope set. -1. Making the variable [masked](../ci/variables/index.md#mask-a-cicd-variable) makes sure it -doesn't show up in the job logs. The variable cannot be -[protected](../ci/variables/index.md#protect-a-cicd-variable), as it needs -to be present for all feature branches. +1. Create a [personal API token](https://gitlab.com/-/profile/personal_access_tokens?name=GitLab+Dangerbot&scopes=api) + that has the `api` scope set (don't forget to copy it to the clipboard). +1. In your fork, add a [project CI/CD variable](../ci/variables/index.md#add-a-cicd-variable-to-a-project) + called `DANGER_GITLAB_API_TOKEN` with the token copied in the previous step. +1. Make the variable [masked](../ci/variables/index.md#mask-a-cicd-variable) so it + doesn't show up in the job logs. The variable cannot be + [protected](../ci/variables/index.md#protected-cicd-variables), because it needs + to be present for all branches. diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index ad2768397e6..3cf9ab1ab5c 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -68,10 +68,72 @@ In this example, the change to ignore the column went into release 12.5. Continuing our example, dropping the column goes into a _post-deployment_ migration in release 12.6: +Start by creating the **post-deployment migration**: + +```shell +bundle exec rails g post_deployment_migration remove_users_updated_at_column +``` + +There are two scenarios that you need to consider +to write a migration that removes a column: + +#### A. The removed column has no indexes or constraints that belong to it + +In this case, a **transactional migration** can be used. Something as simple as: + +```ruby +class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.0] + def up + remove_column :users, :updated_at + end + + def down + add_column :users, :updated_at, :datetime + end +end +``` + +You can consider [enabling lock retries]( +https://docs.gitlab.com/ee/development/migration_style_guide.html#usage-with-transactional-migrations +) when you run a migration on big tables, because it might take some time to +acquire a lock on this table. + +#### B. The removed column has an index or constraint that belongs to it + +If the `down` method requires adding back any dropped indexes or constraints, that cannot +be done within a transactional migration, then the migration would look like this: + ```ruby - remove_column :user, :updated_at +class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + remove_column :users, :updated_at + end + + def down + unless column_exists?(:users, :updated_at) + add_column :users, :updated_at, :datetime + end + + # Make sure to add back any indexes or constraints, + # that were dropped in the `up` method. For example: + add_concurrent_index(:users, :updated_at) + end +end ``` +In the `down` method, we check to see if the column already exists before adding it again. +We do this because the migration is non-transactional and might have failed while it was running. + +The [`disable_ddl_transaction!`]( +https://docs.gitlab.com/ee/development/migration_style_guide.html#usage-with-non-transactional-migrations-disable_ddl_transaction +) is used to disable the transaction that wraps the whole migration. + +You can refer to the page [Migration Style Guide]( +https://docs.gitlab.com/ee/development/migration_style_guide.html +) for more information about database migrations. + ### Step 3: Removing the ignore rule (release M+2) With the next release, in this example 12.7, we set up another merge request to remove the ignore rule. @@ -272,7 +334,7 @@ Renaming a table is possible without downtime by following our multi-release Adding foreign keys usually works in 3 steps: 1. Start a transaction -1. Run `ALTER TABLE` to add the constraint(s) +1. Run `ALTER TABLE` to add the constraints 1. Check all existing data Because `ALTER TABLE` typically acquires an exclusive lock until the end of a diff --git a/doc/development/database/background_migrations.md b/doc/development/database/background_migrations.md index 1f7e0d76c89..80ba0336bda 100644 --- a/doc/development/database/background_migrations.md +++ b/doc/development/database/background_migrations.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Background migrations WARNING: -Background migrations are strongly discouraged in favor of the new [batched background migrations framework](../batched_background_migrations.md). +Background migrations are strongly discouraged in favor of the new [batched background migrations framework](batched_background_migrations.md). Please check that documentation and determine if that framework suits your needs and fall back to these only if required. @@ -45,13 +45,17 @@ into this category. ## Isolation Background migrations must be isolated and can not use application code (for example, -models defined in `app/models`). Since these migrations can take a long time to -run it's possible for new versions to be deployed while they are still running. +models defined in `app/models` except the `ApplicationRecord` classes). Since these migrations +can take a long time to run it's possible for new versions to be deployed while they are still running. It's also possible for different migrations to be executed at the same time. This means that different background migrations should not migrate data in a way that would cause conflicts. +## Accessing data for multiple databases + +See [Accessing data for multiple databases of Batched Background Migrations](batched_background_migrations.md#accessing-data-for-multiple-databases) for more details. + ## Idempotence Background migrations are executed in a context of a Sidekiq process. @@ -190,7 +194,7 @@ class: ```ruby class Gitlab::BackgroundMigration::ExtractIntegrationsUrl - class Integration < ActiveRecord::Base + class Integration < ::ApplicationRecord self.table_name = 'integrations' end @@ -214,7 +218,7 @@ created and updated integrations. We can do this using something along the lines the following: ```ruby -class Integration < ActiveRecord::Base +class Integration < ::ApplicationRecord after_commit :schedule_integration_migration, on: :update after_commit :schedule_integration_migration, on: :create diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md new file mode 100644 index 00000000000..3a0fa77eff9 --- /dev/null +++ b/doc/development/database/batched_background_migrations.md @@ -0,0 +1,371 @@ +--- +type: reference, dev +stage: Enablement +group: Database +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" +--- + +# Batched background migrations + +Batched Background Migrations should be used to perform data migrations whenever a +migration exceeds [the time limits](../migration_style_guide.md#how-long-a-migration-should-take) +in our guidelines. For example, you can use batched background +migrations to migrate data that's stored in a single JSON column +to a separate table instead. + +## When to use batched background migrations + +Use a batched background migration when you migrate _data_ in tables containing +so many rows that the process would exceed +[the time limits in our guidelines](../migration_style_guide.md#how-long-a-migration-should-take) +if performed using a regular Rails migration. + +- Batched background migrations should be used when migrating data in + [high-traffic tables](../migration_style_guide.md#high-traffic-tables). +- Batched background migrations may also be used when executing numerous single-row queries + for every item on a large dataset. Typically, for single-record patterns, runtime is + largely dependent on the size of the dataset. Split the dataset accordingly, + and put it into background migrations. +- Don't use batched background migrations to perform schema migrations. + +Background migrations can help when: + +- Migrating events from one table to multiple separate tables. +- Populating one column based on JSON stored in another column. +- Migrating data that depends on the output of external services. (For example, an API.) + +NOTE: +If the batched background migration is part of an important upgrade, it must be announced +in the release post. Discuss with your Project Manager if you're unsure if the migration falls +into this category. + +## Isolation + +Batched background migrations must be isolated and can not use application code (for example, +models defined in `app/models` except the `ApplicationRecord` classes). +Because these migrations can take a long time to run, it's possible +for new versions to deploy while the migrations are still running. + +## Accessing data for multiple databases + +Background Migration contrary to regular migrations does have access to multiple databases +and can be used to efficiently access and update data across them. To properly indicate +a database to be used it is desired to create ActiveRecord model inline the migration code. +Such model should use a correct [`ApplicationRecord`](multiple_databases.md#gitlab-schema) +depending on which database the table is located. As such usage of `ActiveRecord::Base` +is disallowed as it does not describe a explicitly database to be used to access given table. + +```ruby +# good +class Gitlab::BackgroundMigration::ExtractIntegrationsUrl + class Project < ::ApplicationRecord + self.table_name = 'projects' + end + + class Build < ::Ci::ApplicationRecord + self.table_name = 'ci_builds' + end +end + +# bad +class Gitlab::BackgroundMigration::ExtractIntegrationsUrl + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + class Build < ActiveRecord::Base + self.table_name = 'ci_builds' + end +end +``` + +Similarly the usage of `ActiveRecord::Base.connection` is disallowed and needs to be +replaced preferably with the usage of model connection. + +```ruby +# good +Project.connection.execute("SELECT * FROM projects") + +# acceptable +ApplicationRecord.connection.execute("SELECT * FROM projects") + +# bad +ActiveRecord::Base.connection.execute("SELECT * FROM projects") +``` + +## Idempotence + +Batched background migrations are executed in a context of a Sidekiq process. +The usual Sidekiq rules apply, especially the rule that jobs should be small +and idempotent. Make sure that in case that your migration job is retried, data +integrity is guaranteed. + +See [Sidekiq best practices guidelines](https://github.com/mperham/sidekiq/wiki/Best-Practices) +for more details. + +## Batched background migrations for EE-only features + +All the background migration classes for EE-only features should be present in GitLab CE. +For this purpose, create an empty class for GitLab CE, and extend it for GitLab EE +as explained in the guidelines for +[implementing Enterprise Edition features](../ee_features.md#code-in-libgitlabbackground_migration). + +Batched Background migrations are simple classes that define a `perform` method. A +Sidekiq worker then executes such a class, passing any arguments to it. All +migration classes must be defined in the namespace +`Gitlab::BackgroundMigration`. Place the files in the directory +`lib/gitlab/background_migration/`. + +## Queueing + +Queueing a batched background migration should be done in a post-deployment +migration. Use this `queue_batched_background_migration` example, queueing the +migration to be executed in batches. Replace the class name and arguments with the values +from your migration: + +```ruby +queue_batched_background_migration( + JOB_CLASS_NAME, + TABLE_NAME, + JOB_ARGUMENTS, + JOB_INTERVAL + ) +``` + +Make sure the newly-created data is either migrated, or +saved in both the old and new version upon creation. Removals in +turn can be handled by defining foreign keys with cascading deletes. + +### Requeuing batched background migrations + +If one of the batched background migrations contains a bug that is fixed in a patch +release, you must requeue the batched background migration so the migration +repeats on systems that already performed the initial migration. + +When you requeue the batched background migration, turn the original +queuing into a no-op by clearing up the `#up` and `#down` methods of the +migration performing the requeuing. Otherwise, the batched background migration is +queued multiple times on systems that are upgrading multiple patch releases at +once. + +When you start the second post-deployment migration, delete the +previously batched migration with the provided code: + +```ruby +Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION_NAME, TABLE_NAME, COLUMN, JOB_ARGUMENTS) + .delete_all +``` + +## Cleaning up + +NOTE: +Cleaning up any remaining background migrations must be done in either a major +or minor release. You must not do this in a patch release. + +Because background migrations can take a long time, you can't immediately clean +things up after queueing them. For example, you can't drop a column used in the +migration process, as jobs would fail. You must add a separate _post-deployment_ +migration in a future release that finishes any remaining +jobs before cleaning things up. (For example, removing a column.) + +To migrate the data from column `foo` (containing a big JSON blob) to column `bar` +(containing a string), you would: + +1. Release A: + 1. Create a migration class that performs the migration for a row with a given ID. + 1. Update new rows using one of these techniques: + - Create a new trigger for simple copy operations that don't need application logic. + - Handle this operation in the model/service as the records are created or updated. + - Create a new custom background job that updates the records. + 1. Queue the batched background migration for all existing rows in a post-deployment migration. +1. Release B: + 1. Add a post-deployment migration that checks if the batched background migration is completed. + 1. Deploy code so that the application starts using the new column and stops to update new records. + 1. Remove the old column. + +Bump to the [import/export version](../../user/project/settings/import_export.md) may +be required, if importing a project from a prior version of GitLab requires the +data to be in the new format. + +## Example + +The `routes` table has a `source_type` field that's used for a polymorphic relationship. +As part of a database redesign, we're removing the polymorphic relationship. One step of +the work will be migrating data from the `source_id` column into a new singular foreign key. +Because we intend to delete old rows later, there's no need to update them as part of the +background migration. + +1. Start by defining our migration class, which should inherit + from `Gitlab::BackgroundMigration::BatchedMigrationJob`: + + ```ruby + class Gitlab::BackgroundMigration::BackfillRouteNamespaceId < BatchedMigrationJob + # For illustration purposes, if we were to use a local model we could + # define it like below, using an `ApplicationRecord` as the base class + # class Route < ::ApplicationRecord + # self.table_name = 'routes' + # end + + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { relation.where("source_type <> 'UnusedType'") } + ) do |sub_batch| + sub_batch.update_all('namespace_id = source_id') + end + end + end + ``` + + NOTE: + Job classes must be subclasses of `BatchedMigrationJob` to be + correctly handled by the batched migration framework. Any subclass of + `BatchedMigrationJob` will be initialized with necessary arguments to + execute the batch, as well as a connection to the tracking database. + Additional `job_arguments` set on the migration will be passed to the + job's `perform` method. + +1. Add a new trigger to the database to update newly created and updated routes, + similar to this example: + + ```ruby + execute(<<~SQL) + CREATE OR REPLACE FUNCTION example() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW."namespace_id" = NEW."source_id" + RETURN NEW; + END; + $$; + SQL + ``` + +1. Create a post-deployment migration that queues the migration for existing data: + + ```ruby + class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + MIGRATION = 'BackfillRouteNamespaceId' + DELAY_INTERVAL = 2.minutes + + def up + queue_batched_background_migration( + MIGRATION, + :routes, + :id, + job_interval: DELAY_INTERVAL + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :routes, :id, []).delete_all + end + end + ``` + + After deployment, our application: + - Continues using the data as before. + - Ensures that both existing and new data are migrated. + +1. In the next release, remove the trigger. We must also add a new post-deployment migration + that checks that the batched background migration is completed. For example: + + ```ruby + class FinalizeBackfillRouteNamespaceId < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillRouteNamespaceId' + disable_ddl_transaction! + + def up + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: :routes, + column_name: :id, + job_arguments: [] + ) + end + + def down + # no-op + end + end + ``` + + If the application does not depend on the data being 100% migrated (for + instance, the data is advisory, and not mission-critical), then you can skip this + final step. This step confirms that the migration is completed, and all of the rows were migrated. + +After the batched migration is completed, you can safely depend on the +data in `routes.namespace_id` being populated. + +## Testing + +Writing tests is required for: + +- The batched background migrations' queueing migration. +- The batched background migration itself. +- A cleanup migration. + +The `:migration` and `schema: :latest` RSpec tags are automatically set for +background migration specs. Refer to the +[Testing Rails migrations](../testing_guide/testing_migrations_guide.md#testing-a-non-activerecordmigration-class) +style guide. + +Remember that `before` and `after` RSpec hooks +migrate your database down and up. These hooks can result in other batched background +migrations being called. Using `spy` test doubles with +`have_received` is encouraged, instead of using regular test doubles, because +your expectations defined in a `it` block can conflict with what is +called in RSpec hooks. Refer to [issue #35351](https://gitlab.com/gitlab-org/gitlab/-/issues/18839) +for more details. + +## Best practices + +1. Know how much data you're dealing with. +1. Make sure the batched background migration jobs are idempotent. +1. Confirm the tests you write are not false positives. +1. If the data being migrated is critical and cannot be lost, the + clean-up migration must also check the final state of the data before completing. +1. Discuss the numbers with a database specialist. The migration may add + more pressure on DB than you expect. Measure on staging, + or ask someone to measure on production. +1. Know how much time is required to run the batched background migration. + +## Additional tips and strategies + +### Viewing failure error logs + +You can view failures in two ways: + +- Via GitLab logs: + 1. After running a batched background migration, if any jobs fail, + view the logs in [Kibana](https://log.gprd.gitlab.net/goto/5f06a57f768c6025e1c65aefb4075694). + View the production Sidekiq log and filter for: + + - `json.new_state: failed` + - `json.job_class_name: <Batched Background Migration job class name>` + - `json.job_arguments: <Batched Background Migration job class arguments>` + + 1. Review the `json.exception_class` and `json.exception_message` values to help + understand why the jobs failed. + + 1. Remember the retry mechanism. Having a failure does not mean the job failed. + Always check the last status of the job. + +- Via database: + + 1. Get the batched background migration `CLASS_NAME`. + 1. Execute the following query in the PostgreSQL console: + + ```sql + SELECT migration.id, migration.job_class_name, transition_logs.exception_class, transition_logs.exception_message + FROM batched_background_migrations as migration + INNER JOIN batched_background_migration_jobs as jobs + ON jobs.batched_background_migration_id = migration.id + INNER JOIN batched_background_migration_job_transition_logs as transition_logs + ON transition_logs.batched_background_migration_job_id = jobs.id + WHERE transition_logs.next_status = '2' AND migration.job_class_name = "CLASS_NAME"; + ``` diff --git a/doc/development/database/loose_foreign_keys.md b/doc/development/database/loose_foreign_keys.md index 2bcdc91202a..3db24793f1b 100644 --- a/doc/development/database/loose_foreign_keys.md +++ b/doc/development/database/loose_foreign_keys.md @@ -117,8 +117,8 @@ Showing cross-schema foreign keys (20): 18 | N | ci_job_token_project_scope_links | projects | target_project_id | cascade 19 | N | ci_project_monthly_usages | projects | project_id | cascade -To match FK write one or many filters to match against FROM/TO/COLUMN: -- scripts/decomposition/generate-loose-foreign-key <filter(s)...> +To match foreign key (FK), write one or many filters to match against FROM/TO/COLUMN: +- scripts/decomposition/generate-loose-foreign-key (filters...) - scripts/decomposition/generate-loose-foreign-key ci_job_artifacts project_id - scripts/decomposition/generate-loose-foreign-key dast_site_profiles_pipelines ``` @@ -593,7 +593,7 @@ Partitions: gitlab_partitions_dynamic.loose_foreign_keys_deleted_records_84 FOR The `partition` column controls the insert direction, the `partition` value determines which partition will get the deleted rows inserted via the trigger. Notice that the default value of the `partition` table matches with the value of the list partition (84). In `INSERT` query -within the trigger thevalue of the `partition` is omitted, the trigger always relies on the +within the trigger the value of the `partition` is omitted, the trigger always relies on the default value of the column. Example `INSERT` query for the trigger: @@ -605,7 +605,7 @@ SELECT TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, old_table.id FROM old_table; ``` The partition "sliding" process is controlled by two, regularly executed callbacks. These -callbackes are defined within the `LooseForeignKeys::DeletedRecord` model. +callbacks are defined within the `LooseForeignKeys::DeletedRecord` model. The `next_partition_if` callback controls when to create a new partition. A new partition will be created when the current partition has at least one record older than 24 hours. A new partition @@ -805,7 +805,7 @@ Possible solutions: - Long-term: invoke the worker more frequently. Parallelize the worker For a one-time fix, we can run the cleanup worker several times from the rails console. The worker -can run parallelly however, this can introduce lock contention and it could increase the worker +can run in parallel however, this can introduce lock contention and it could increase the worker runtime. ```ruby diff --git a/doc/development/database/migrations_for_multiple_databases.md b/doc/development/database/migrations_for_multiple_databases.md index 0ec4612e985..ce326a6ce4a 100644 --- a/doc/development/database/migrations_for_multiple_databases.md +++ b/doc/development/database/migrations_for_multiple_databases.md @@ -33,7 +33,7 @@ Depending on the used constructs, we can classify migrations to be either: Migrations cannot mix **DDL** and **DML** changes as the application requires the structure (as described by `db/structure.sql`) to be exactly the same across all decomposed databases. -### Data Definition Language (DDL) +### Data Definition Language (DDL) The DDL migrations are all migrations that: @@ -43,7 +43,7 @@ The DDL migrations are all migrations that: 1. Add or remove a column with or without a default value (for example, `add_column`). 1. Create or drop trigger functions (for example, `create_trigger_function`). 1. Attach or detach triggers from tables (for example, `track_record_deletions`, `untrack_record_deletions`). -1. Prepare or not async indexes (for example, `prepare_async_index`, `unprepare_async_index_by_name`). +1. Prepare or not asynchronous indexes (for example, `prepare_async_index`, `unprepare_async_index_by_name`). As such DDL migrations **CANNOT**: @@ -159,7 +159,7 @@ end ### The special purpose of `gitlab_shared` -As described in [gitlab_schema](multiple_databases.md#the-special-purpose-of-gitlab_shared), +As described in [`gitlab_schema`](multiple_databases.md#the-special-purpose-of-gitlab_shared), the `gitlab_shared` tables are allowed to contain data across all databases. This implies that such migrations should run across all databases to modify structure (DDL) or modify data (DML). @@ -388,3 +388,32 @@ A Potential extension is to limit running DML migration only to specific environ ```ruby restrict_gitlab_migration gitlab_schema: :gitlab_main, gitlab_env: :gitlab_com ``` + +## Background migrations + +When you use: + +- Background migrations with `track_jobs` set to `true` or +- Batched background migrations + +The migration has to write to a jobs table. All of the +jobs tables used by background migrations are marked as `gitlab_shared`. +You can use these migrations when migrating tables in any database. + +However, when queuing the batches, you must set `restrict_gitlab_migration` based on the +table you are iterating over. If you are updating all `projects`, for example, then you would set +`restrict_gitlab_migration gitlab_schema: :gitlab_main`. If, however, you are +updating all `ci_pipelines`, you would set +`restrict_gitlab_migration gitlab_schema: :gitlab_ci`. + +As with all DML migrations, you cannot query another database outside of +`restrict_gitlab_migration` or `gitlab_shared`. If you need to query another database, +you'll likely need to separate these into two migrations somehow. + +Because the actual migration logic (not the queueing step) for background +migrations runs in a Sidekiq worker, the logic can perform DML queries on +tables in any database, just like any ordinary Sidekiq worker can. + +## How to determine `gitlab_schema` for a given table + +See [GitLab Schema](multiple_databases.md#gitlab-schema). diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index 3b1b06b557c..c622d4f50ff 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -74,7 +74,14 @@ in GitLab 14.1. This feature is still under development, and is not ready for pr ### Configure single database -By default, GDK is configured to run with multiple databases. To configure GDK to use a single database: +By default, GDK is configured to run with multiple databases. + +WARNING: +Switching back-and-forth between single and multiple databases in +the same development instance is discouraged. Any data in the `ci` +database will not be accessible in single database mode. For single database, you should use a separate development instance. + +To configure GDK to use a single database: 1. On the GDK root directory, run: @@ -519,7 +526,7 @@ ci_build.update!(updated_at: Time.current) # CI DB ci_build.project.update!(updated_at: Time.current) # Main DB ``` -##### Async processing +##### Asynchronous processing If we need more guarantee that an operation finishes the work consistently we can execute it within a background job. A background job is scheduled asynchronously and retried several times @@ -579,58 +586,6 @@ ensures that we forbid destroying the parent object if something is not cleaned If all you need to do is clean up the child records themselves from PostgreSQL, consider using [loose foreign keys](loose_foreign_keys.md). -## `config/database.yml` - -GitLab is adding support to run multiple databases, for example to -[separate tables for the continuous integration features](https://gitlab.com/groups/gitlab-org/-/epics/6167) -from the main database. In order to prepare for this change, we -[validate the structure of the configuration](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67877) -in `database.yml` to ensure that only known databases are used. - -Previously, the `config/database.yml` looked like this: - -```yaml -production: - adapter: postgresql - encoding: unicode - database: gitlabhq_production - ... -``` - -With the support for many databases this -syntax is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/338182) -and will be removed in [15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/338182). - -The new `config/database.yml` needs to include a database name -to define a database configuration. Only `main:` and `ci:` database -names are supported. The `main:` database must always be a first -entry in a hash. This change applies to decomposed and non-decomposed -change. If an invalid or deprecated syntax is used the error -or warning is printed during application start. - -```yaml -# Non-decomposed database -production: - main: - adapter: postgresql - encoding: unicode - database: gitlabhq_production - ... - -# Decomposed database -production: - main: - adapter: postgresql - encoding: unicode - database: gitlabhq_production - ... - ci: - adapter: postgresql - encoding: unicode - database: gitlabhq_production_ci - ... -``` - ## Foreign keys that cross databases There are many places where we use foreign keys that reference across the two diff --git a/doc/development/database/pagination_guidelines.md b/doc/development/database/pagination_guidelines.md index 3a772b10a6d..08840124535 100644 --- a/doc/development/database/pagination_guidelines.md +++ b/doc/development/database/pagination_guidelines.md @@ -172,7 +172,7 @@ From the user point of view, this might not be always noticeable. As the user pa When requesting a large page number, the database needs to read `PAGE * PAGE_SIZE` rows. This makes offset pagination **unsuitable for large database tables**. -Example: listing users on the Admin page +Example: listing users on the Admin Area Listing users with a very simple SQL query: diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index 4ed7cf1b4de..7aa529e1518 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -206,7 +206,7 @@ class ScheduleCapTitleLengthOnIssues < Gitlab::Database::Migration[1.0] disable_ddl_transaction! - class Issue < ActiveRecord::Base + class Issue < ::ApplicationRecord include EachBatch self.table_name = 'issues' diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md index ec768136404..34cb73978bc 100644 --- a/doc/development/database/table_partitioning.md +++ b/doc/development/database/table_partitioning.md @@ -43,7 +43,7 @@ problem. First, a table is partitioned on a partition key, which is a column or set of columns which determine how the data will be split across the partitions. The partition key is used by the database when reading or -writing data, to decide which partition(s) need to be accessed. The +writing data, to decide which partitions need to be accessed. The partition key should be a column that would be included in a `WHERE` clause on almost all queries accessing that table. diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md index 08e29e373f6..cafc40ccc68 100644 --- a/doc/development/deprecation_guidelines/index.md +++ b/doc/development/deprecation_guidelines/index.md @@ -21,8 +21,6 @@ deprecated. ## When can a feature be deprecated? -A feature can be deprecated at any time, provided there is a viable alternative. - Deprecations should be announced on the [Deprecated feature removal schedule](../../update/deprecations.md). For steps to create a deprecation entry, see [Deprecations](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations). @@ -37,3 +35,52 @@ For API removals, see the [GraphQL](../../api/graphql/index.md#deprecation-and-r For configuration removals, see the [Omnibus deprecation policy](../../administration/package_information/deprecation_policy.md). For versioning and upgrade details, see our [Release and Maintenance policy](../../policy/maintenance.md). + +## Update the deprecations and removals documentation + +The [deprecations](../../update/deprecations.md) and [removals](../../update/removals.md) +documentation is generated from the YAML files located in +[`gitlab/data/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/data). + +To update the deprecations and removals pages when an entry is added, +edited, or removed: + +1. From the command line, navigate to your local clone of the [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab) project. +1. Create, edit, or remove the YAML file under [deprecations](https://gitlab.com/gitlab-org/gitlab/-/tree/master/data/deprecations) + or [removals](https://gitlab.com/gitlab-org/gitlab/-/tree/master/data/removals). +1. Compile the deprecation or removals documentation with the appropriate command: + + - For deprecations: + + ```shell + bin/rake gitlab:docs:compile_deprecations + ``` + + - For removals: + + ```shell + bin/rake gitlab:docs:compile_removals + ``` + +1. If needed, you can verify the docs are up to date with: + + - For deprecations: + + ```shell + bin/rake gitlab:docs:check_deprecations + ``` + + - For removals: + + ```shell + bin/rake gitlab:docs:check_removals + ``` + +1. Commit the updated documentation and push the changes. +1. Create a merge request using the [Deprecations](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Deprecations.md) + or [Removals](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Removals.md) templates. + +Related Handbook pages: + +- <https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations-removals-and-breaking-changes> +- <https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-and-removals-docs> diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 680ac71f857..b4f347449cc 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -71,9 +71,7 @@ GITLAB_TRACING=opentracing://<driver>?<param_name>=<param_value>&<param_name_2>= In this example, we have the following hypothetical values: -- `driver`: the driver. [GitLab supports - `jaeger`](../operations/tracing.md). In future, other - tracing implementations may also be supported. +- `driver`: the driver such a jaegar. - `param_name`, `param_value`: these are driver specific configuration values. Configuration parameters for Jaeger are documented [further on in this document](#2-configure-the-gitlab_tracing-environment-variable) they should be URL encoded. diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index fb58851e93f..c5ea1985fc7 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -45,7 +45,7 @@ You can combine entries if they happened in the same release: ## Use a note to describe the state of the feature flag -Information about feature flags should be in a **Note** at the start of the topic (just below the version history). +Information about feature flags should be in a `FLAG` note at the start of the topic (just below the version history). The note has three parts, and follows this structure: @@ -62,6 +62,7 @@ FLAG: |--------------------------|---------------| | Available | `On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | | Unavailable | `On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | +| Available to some users | `On self-managed GitLab, by default this feature is available to a subset of users. To show or hide the feature for all, ask an administrator to [change the status of the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | | Available, per-group | `On self-managed GitLab, by default this feature is available. To hide the feature per group, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | | Unavailable, per-group | `On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | | Available, per-project | `On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | @@ -71,11 +72,11 @@ FLAG: ### GitLab.com availability information -| If the feature is... | Use this text | -|-------------------------------------|---------------| -| Available | `On GitLab.com, this feature is available.` | -| Available to GitLab.com admins only | `On GitLab.com, this feature is available but can be configured by GitLab.com administrators only.` -| Unavailable | `On GitLab.com, this feature is not available.`| +| If the feature is... | Use this text | +|---------------------------------------------|---------------| +| Available | `On GitLab.com, this feature is available.` | +| Available to GitLab.com administrators only | `On GitLab.com, this feature is available but can be configured by GitLab.com administrators only.` +| Unavailable | `On GitLab.com, this feature is not available.`| ### Optional information diff --git a/doc/development/documentation/restful_api_styleguide.md b/doc/development/documentation/restful_api_styleguide.md index 8a505ed84a8..0a24f9b67be 100644 --- a/doc/development/documentation/restful_api_styleguide.md +++ b/doc/development/documentation/restful_api_styleguide.md @@ -26,11 +26,12 @@ In the Markdown doc for a resource (AKA endpoint): GET /projects/:id/repository/branches ``` -- Every method must have a detailed [description of the parameters](#method-description). +- Every method must have a detailed [description of the attributes](#method-description). - Every method must have a cURL example. -- Every method must have a response body (in JSON format). +- Every method must have a detailed [description of the response body](#response-body-description). +- Every method must have a response body example (in JSON format). - If an attribute is available only to higher level tiers than the other - parameters, add the appropriate inline [tier badge](styleguide/index.md#product-tier-badges). + attributes, add the appropriate inline [tier badge](styleguide/index.md#product-tier-badges). Put the badge in the **Attribute** column, like the `**(<tier>)**` code in the following template. @@ -59,6 +60,13 @@ Supported attributes: | `attribute` | datatype | **{dotted-circle}** No | Detailed description. | | `attribute` | datatype | **{dotted-circle}** No | Detailed description. | +Response body attributes: + +| Attribute | Type | Description | +|:-------------------------|:---------|:----------------------| +| `attribute` | datatype | Detailed description. | +| `attribute` **(<tier>)** | datatype | Detailed description. | + Example request: ```shell @@ -75,7 +83,7 @@ Example response: ``` ```` -Adjust the [version history note accordingly](styleguide/index.md#version-text-in-the-version-history) +Adjust the [version history note accordingly](versions.md#add-a-version-history-item) to describe the GitLab release that introduced the API call. ## Method description @@ -86,23 +94,51 @@ always be in code blocks using backticks (`` ` ``). Sort the table by required attributes first, then alphabetically. ```markdown -| Attribute | Type | Required | Description | -|:-----------------------------|:--------------|:-----------------------|:-----------------------------------------------------| +| Attribute | Type | Required | Description | +|:-----------------------------|:--------------|:-----------------------|:----------------------------------------------------| | `title` | string | **{check-circle}** Yes | Title of the issue. | -| `assignee_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | IDs of the users to assign the issue to. | +| `assignee_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | IDs of the users to assign the issue to. | | `confidential` | boolean | **{dotted-circle}** No | Sets the issue to confidential. Default is `false`. | ``` Rendered example: -| Attribute | Type | Required | Description | -|:-----------------------------|:--------------|:-----------------------|:-----------------------------------------------------| +| Attribute | Type | Required | Description | +|:-----------------------------|:--------------|:-----------------------|:----------------------------------------------------| | `title` | string | **{check-circle}** Yes | Title of the issue. | -| `assignee_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | IDs of the users to assign the issue to. | +| `assignee_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | IDs of the users to assign the issue to. | | `confidential` | boolean | **{dotted-circle}** No | Sets the issue to confidential. Default is `false`. | For information about writing attribute descriptions, see the [GraphQL API description style guide](../api_graphql_styleguide.md#description-style-guide). +## Response body description + +Use the following table headers to describe the response bodies. Attributes should +always be in code blocks using backticks (`` ` ``). + +If the attribute is a complex type, like another object, represent sub-attributes +with dots (`.`), like `project.name` or `projects[].name` in case of an array. + +Sort the table alphabetically. + +```markdown +| Attribute | Type | Description | +|:-----------------------------|:--------------|:------------------------------------------| +| `assignee_ids` **(PREMIUM)** | integer array | IDs of the users to assign the issue to. | +| `confidential` | boolean | Whether the issue is confidential or not. | +| `title` | string | Title of the issue. | +``` + +Rendered example: + +| Attribute | Type | Description | +|:-----------------------------|:--------------|:------------------------------------------| +| `assignee_ids` **(PREMIUM)** | integer array | IDs of the users to assign the issue to. | +| `confidential` | boolean | Whether the issue is confidential or not. | +| `title` | string | Title of the issue. | + +For information about writing attribute descriptions, see the [GraphQL API description style guide](../api_graphql_styleguide.md#description-style-guide). + ## cURL commands - Use `https://gitlab.example.com/api/v4/` as an endpoint. @@ -116,9 +152,9 @@ For information about writing attribute descriptions, see the [GraphQL API descr | Methods | Description | |:------------------------------------------------|:-------------------------------------------------------| | `--header "PRIVATE-TOKEN: <your_access_token>"` | Use this method as is, whenever authentication needed. | -| `--request POST` | Use this method when creating new objects | -| `--request PUT` | Use this method when updating existing objects | -| `--request DELETE` | Use this method when removing existing objects | +| `--request POST` | Use this method when creating new objects. | +| `--request PUT` | Use this method when updating existing objects. | +| `--request DELETE` | Use this method when removing existing objects. | ## cURL Examples diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index bdda15e2064..3566ab82379 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -22,25 +22,29 @@ from where content is sourced, the `gitlab-docs` project, and the published outp ```mermaid graph LR - A[gitlab/doc] - B[gitlab-runner/docs] - C[omnibus-gitlab/doc] - D[charts/doc] - E[gitlab-docs] - A --> E - B --> E - C --> E - D --> E - E -- Build pipeline --> F - F[docs.gitlab.com] - H[/ee/] - I[/runner/] - J[/omnibus/] - K[/charts/] - F --> H - F --> I - F --> J - F --> K + A[gitlab-org/gitlab/doc] + B[gitlab-org/gitlab-runner/docs] + C[gitlab-org/omnibus-gitlab/doc] + D[gitlab-org/charts/gitlab/doc] + E[gitlab-org/cloud-native/gitlab-operator/doc] + Y[gitlab-org/gitlab-docs] + A --> Y + B --> Y + C --> Y + D --> Y + E --> Y + Y -- Build pipeline --> Z + Z[docs.gitlab.com] + M[//ee/] + N[//runner/] + O[//omnibus/] + P[//charts/] + Q[//operator/] + Z --> M + Z --> N + Z --> O + Z --> P + Z --> Q ``` GitLab docs content isn't kept in the `gitlab-docs` repository. @@ -48,9 +52,10 @@ All documentation files are hosted in the respective repository of each product, and all together are pulled to generate the docs website: - [GitLab](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) -- [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/tree/master/doc) +- [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/tree/master/doc) - [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/-/tree/main/docs) -- [GitLab Chart](https://gitlab.com/charts/gitlab/tree/master/doc) +- [GitLab Chart](https://gitlab.com/gitlab-org/charts/gitlab/-/tree/master/doc) +- [GitLab Operator](https://gitlab.com/gitlab-org/cloud-native/gitlab-operator/-/tree/master/doc) Learn more about [the docs folder structure](folder_structure.md). @@ -231,31 +236,9 @@ If you don't specify `editor:`, the simple one is used by default. ## Algolia search engine The docs site uses [Algolia DocSearch](https://community.algolia.com/docsearch/) -for its search function. This is how it works: - -1. GitLab is a member of the [DocSearch program](https://community.algolia.com/docsearch/#join-docsearch-program), - which is the free tier of [Algolia](https://www.algolia.com/). -1. Algolia hosts a [DocSearch configuration](https://github.com/algolia/docsearch-configs/blob/master/configs/gitlab.json) - for the GitLab docs site, and we've worked together to refine it. -1. That [configuration](https://community.algolia.com/docsearch/config-file.html) is - parsed by their [crawler](https://community.algolia.com/docsearch/crawler-overview.html) - every 24h and [stores](https://community.algolia.com/docsearch/inside-the-engine.html) - the [DocSearch index](https://community.algolia.com/docsearch/how-do-we-build-an-index.html) - on [Algolia's servers](https://community.algolia.com/docsearch/faq.html#where-is-my-data-hosted%3F). -1. On the docs side, we use a [DocSearch layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/layouts/docsearch.html) which - is present on pretty much every page except <https://docs.gitlab.com/search/>, - which uses its [own layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/layouts/instantsearch.html). In those layouts, - there's a JavaScript snippet which initiates DocSearch by using an API key - and an index name (`gitlab`) that are needed for Algolia to show the results. - -### Algolia notes for GitLab team members - -If you're a GitLab team member, find credentials for the Algolia dashboard -in the shared [GitLab 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams). -To receive weekly reports of the search usage, search the Google doc with -title `Email, Slack, and GitLab Groups and Aliases`, search for `docsearch`, -and add a comment with your email to be added to the alias that gets the weekly -reports. +for its search function. + +Learn more in <https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/docsearch.md>. ## Monthly release process (versions) diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index 21368098f39..329fd279b99 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -237,7 +237,7 @@ consider using subsections for each distinct task. ### Related topics If inline links are not sufficient, you can create a topic called **Related topics** -and include a bulleted list of related topics. This topic should be above the Troubleshooting section. +and include an unordered list of related topics. This topic should be above the Troubleshooting section. ```markdown # Related topics @@ -336,7 +336,7 @@ Consider the following guidelines when offering examples: the reader to go directly to the good part. Consider offering an explanation (for example, a comment, or a link to a resource) on why something is bad practice. -- Better and best cases can be considered part of the good case(s) code block. +- Better and best cases can be considered part of the good cases' code block. In the same code block, precede each with comments: `# Better` and `# Best`. Although the bad-then-good approach is acceptable for the GitLab development diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 7bfc0320d02..c11d1422167 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -276,7 +276,6 @@ You can use these fake tokens as examples: | Trigger token | `be20d8dcc028677c931e04f3871a9b` | | Webhook secret token | `6XhDroRcYPM5by_h-HLY` | | Health check token | `Tu7BgjR9qeZTEyRzGG2P` | -| Request profile token | `7VgpS4Ax5utVD2esNstz` | ### Contractions @@ -401,6 +400,39 @@ Backticks are more precise than quotes. For example, in this string: It's not clear whether the user should include the period in the string. +### Inline code + +Inline code style is applied inline with regular text. Use inline code style: + +- For filenames or fragments of configuration files. For example, `.gitlab-ci.yml`, `CODEOWNERS`, and `only: [main]`. +- For HTTP methods (`HTTP POST`) and HTTP status codes, both full (`404 File Not Found`) and abbreviated (`404`). + For example: Send a `DELETE` request to delete the runner. Send a `POST` request to create one. + +To apply inline code style, wrap the text in a single backtick (`` ` ``). For example, `this is inline code style`. + +### Code blocks + +Code block style separates code text from regular text. Use code block style for commands run in the command-line +interface. Code block style is easier to copy and paste in a user's terminal window. + +To apply code block style, wrap the text in triple backticks (three `` ` ``) and add a syntax highlighting hint. For +example: + +````plaintext +```plaintext +This is codeblock style +``` +```` + +When using code block style: + +- Use quadruple backticks (four `` ` ``) to apply code block style when the code block you are styling has triple + backticks in it. For example, when illustrating code block style. +- Add a blank line above and below code blocks. +- Syntax highlight hints are required for code blocks. See the + [list of supported languages and lexers](https://github.com/rouge-ruby/rouge/wiki/List-of-supported-languages-and-lexers) + for available syntax highlighters. Use `plaintext` if no better hint is available. + ## Lists - Always start list items with a capital letter, unless they're parameters or @@ -622,7 +654,10 @@ In the Markdown document: For the heading text, **do**: - Be clear and direct. Make every word count. -- Use active verbs for tasks. For example, `Configure GDK` instead of `Configuring GDK`. +- Use active, imperative verbs for [tasks](../structure.md#task). For example, `Create an issue`. +- Use `ing` (gerund) verbs only when you need a topic that introduces tasks. For example, `Configuring GDK`. +- Use nouns for [concepts](../structure.md#concept). For example, `GDK dependency management`. If a noun is + ambiguous, you can add a gerund. For example, `Documenting versions` instead of `Versions`. - Talk about what the product does, realistically but from a positive perspective. Instead of `Limitations`, move the content near other similar information. If you must, you can use the title `Known issues`. @@ -695,7 +730,6 @@ We include guidance for links in these categories: for authoritative sources. - When to use [links requiring permissions](#links-requiring-permissions). - How to set up a [link to a video](#link-to-video). -- How to [include links with version text](#where-to-put-version-text). - How to [link to specific lines of code](#link-to-specific-lines-of-code) ### Basic link criteria @@ -949,7 +983,7 @@ If you are documenting multiple fields and only one field needs explanation, do 1. Expand **Push rules**. 1. Complete the fields. **Branch name** must be a regular expression. -To describe multiple fields, use bullets: +To describe multiple fields, use unordered list items: 1. Expand **General pipelines**. 1. Complete the fields. @@ -1166,80 +1200,6 @@ different mobile devices. `/help`, because the GitLab Markdown processor doesn't support iframes. It's hidden on the documentation site, but is displayed by `/help`. -## Code blocks - -- Always wrap code added to a sentence in inline code blocks (`` ` ``). - For example, `.gitlab-ci.yml`, `git add .`, `CODEOWNERS`, or `only: [main]`. - File names, commands, entries, and anything that refers to code should be - added to code blocks. To make things easier for the user, always add a full - code block for things that can be useful to copy and paste, as they can do it - with the button on code blocks. -- HTTP methods (`HTTP POST`) and HTTP status codes, both full (`404 File Not Found`) - and abbreviated (`404`), should be wrapped in inline code blocks when used in sentences. - For example: Send a `DELETE` request to delete the runner. Send a `POST` request to create one. -- Add a blank line above and below code blocks. -- When providing a shell command and its output, prefix the shell command with `$` - and leave a blank line between the command and the output. -- When providing a command without output, don't prefix the shell command with `$`. -- If you need to include triple backticks inside a code block, use four backticks - for the code block fences instead of three. -- For regular fenced code blocks, always use a highlighting class corresponding to - the language for better readability. Examples: - - ````markdown - ```ruby - Ruby code - ``` - - ```javascript - JavaScript code - ``` - - ```markdown - [Markdown code example](example.md) - ``` - - ```plaintext - Code or text for which no specific highlighting class is available. - ``` - ```` - -Syntax highlighting is required for fenced code blocks added to the GitLab -documentation. Refer to this table for the most common language classes, -or check the [complete list](https://github.com/rouge-ruby/rouge/wiki/List-of-supported-languages-and-lexers) -of available language classes: - -| Preferred language tags | Language aliases and notes | -|-------------------------|------------------------------------------------------------------------------| -| `asciidoc` | | -| `dockerfile` | Alias: `docker`. | -| `elixir` | | -| `erb` | | -| `golang` | Alias: `go`. | -| `graphql` | | -| `haml` | | -| `html` | | -| `ini` | For some simple configuration files that are not in TOML format. | -| `javascript` | Alias `js`. | -| `json` | | -| `markdown` | Alias: `md`. | -| `mermaid` | | -| `nginx` | | -| `perl` | | -| `php` | | -| `plaintext` | Examples with no defined language, such as output from shell commands or API calls. If a code block has no language, it defaults to `plaintext`. Alias: `text`.| -| `prometheus` | Prometheus configuration examples. | -| `python` | | -| `ruby` | Alias: `rb`. | -| `shell` | Aliases: `bash` or `sh`. | -| `sql` | | -| `toml` | Runner configuration examples, and other TOML-formatted configuration files. | -| `typescript` | Alias: `ts`. | -| `xml` | | -| `yaml` | Alias: `yml`. | - -For a complete reference on code blocks, see the [Kramdown guide](https://about.gitlab.com/handbook/markdown-guide/#code-blocks). - ## GitLab SVG icons > [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/384) in GitLab 12.7. @@ -1379,7 +1339,7 @@ you don't need to supply your username and password each time. ### Disclaimer Use to describe future functionality only. -For more information, see [Legal disclaimer for future features](#legal-disclaimer-for-future-features). +For more information, see [Legal disclaimer for future features](../versions.md#legal-disclaimer-for-future-features). ## Blockquotes @@ -1429,222 +1389,6 @@ application: - For elements with a tooltip or hover label, use that label in bold with matching case. For example, `Select **Add status emoji**`. -## GitLab versions - -GitLab product documentation pages (not including [Contributor and Development](../../index.md) -pages in the `/development` directory) can include version information to help -users be aware of recent improvements or additions. - -The GitLab Technical Writing team determines which versions of -documentation to display on this site based on the GitLab -[Statement of Support](https://about.gitlab.com/support/statement-of-support.html#version-support). - -### View older GitLab documentation versions - -Older versions of GitLab may no longer have documentation available from `docs.gitlab.com`. -If documentation for your version is no longer available from `docs.gitlab.com`, you can still view a -tagged and released set of documentation for your installed version: - -- In the [documentation archives](https://docs.gitlab.com/archives/). -- At the `/help` URL of your GitLab instance. -- In the documentation repository based on the respective branch (for example, - the [13.2 branch](https://gitlab.com/gitlab-org/gitlab/-/tree/13-2-stable-ee/doc)). - -### Where to put version text - -When a feature is added or updated, you can include its version information -either as a **Version history** item or as an inline text reference. - -#### Version text in the **Version History** - -If all content in a section is related, add version text after the header for -the section. The version information must: - -- Be surrounded by blank lines. -- Start with `>`. If there are multiple bullets, each line must start with `> -`. -- The string must include these words in this order (capitalization doesn't matter): - - `introduced`, `enabled`, `deprecated`, `changed`, `moved`, `recommended` (as in the - [feature flag documentation](../feature_flags.md)), `removed`, or `renamed` - - `in` or `to` - - `GitLab` -- Whenever possible, include a link to the completed issue, merge request, or epic - that introduced the feature. An issue is preferred over a merge request, and - a merge request is preferred over an epic. -- Do not include information about the tier, unless documenting a tier change - (for example, `Feature X [moved](issue-link) to Premium in GitLab 19.2`). -- Do not link to the pricing page. - The tier is provided by the [product badge](#product-tier-badges) on the heading. - -```markdown -## Feature name - -> [Introduced](<link-to-issue>) in GitLab 11.3. - -This feature does something. - -## Feature name 2 - -> - [Introduced](<link-to-issue>) in GitLab 11.3. -> - [Enabled by default](<link-to-issue>) in GitLab 11.4. - -This feature does something else. -``` - -If you're documenting elements of a feature, start with the feature name or a gerund: - -```markdown -> - Notifications for expiring tokens [introduced](<link-to-issue>) in GitLab 11.3. -> - Creating an issue from an issue board [introduced](<link-to-issue>) in GitLab 13.1. -``` - -If a feature is moved to another tier: - -```markdown -> - [Moved](<link-to-issue>) from GitLab Ultimate to GitLab Premium in 11.8. -> - [Moved](<link-to-issue>) from GitLab Premium to GitLab Free in 12.0. -``` - -#### Inline version text - -If you're adding content to an existing topic, you can add version information -inline with the existing text. - -In this case, add `([introduced/deprecated](<link-to-issue>) in GitLab X.X)`. - -Including the issue link is encouraged, but isn't a requirement. For example: - -```markdown -The voting strategy in GitLab 13.4 and later requires the primary and secondary -voters to agree. -``` - -#### Deprecated features - -When a feature is deprecated, add `(DEPRECATED)` to the page title or to -the heading of the section documenting the feature, immediately before -the tier badge: - -```markdown -<!-- Page title example: --> -# Feature A (DEPRECATED) **(ALL TIERS)** - -<!-- Doc section example: --> -## Feature B (DEPRECATED) **(PREMIUM SELF)** -``` - -Add the deprecation to the version history note (you can include a link -to a replacement when available): - -```markdown -> - [Deprecated](<link-to-issue>) in GitLab 11.3. Replaced by [meaningful text](<link-to-appropriate-documentation>). -``` - -You can also describe the replacement in surrounding text, if available. If the -deprecation isn't obvious in existing text, you may want to include a warning: - -```markdown -WARNING: -This feature was [deprecated](link-to-issue) in GitLab 12.3 and replaced by -[Feature name](link-to-feature-documentation). -``` - -If you add `(DEPRECATED)` to the page's title and the document is linked from the docs -navigation, either remove the page from the nav or update the nav item to include the -same text before the feature name: - -```yaml - - doc_title: (DEPRECATED) Feature A -``` - -In the first major GitLab version after the feature was deprecated, be sure to -remove information about that deprecated feature. - -#### End-of-life for features or products - -When a feature or product enters its end-of-life, indicate its status by -creating a [warning alert](#alert-boxes) directly after its relevant header. -If possible, link to its deprecation and removal issues. - -For example: - -```markdown -WARNING: -This feature is in its end-of-life process. It is [deprecated](link-to-issue) -in GitLab X.X, and is planned for [removal](link-to-issue) in GitLab X.X. -``` - -After the feature or product is officially deprecated and removed, remove -its information from the GitLab documentation. - -### Versions in the past or future - -When describing functionality available in past or future versions, use: - -- Earlier, and not older or before. -- Later, and not newer or after. - -For example: - -- Available in GitLab 13.1 and earlier. -- Available in GitLab 12.4 and later. -- In GitLab 12.2 and earlier, ... -- In GitLab 11.6 and later, ... - -### Promising features in future versions - -Do not promise to deliver features in a future release. For example, avoid phrases like, -"Support for this feature is planned." - -We cannot guarantee future feature work, and promises -like these can raise legal issues. Instead, say that an issue exists. -For example: - -- Support for improvements is tracked `[in this issue](LINK)`. -- You cannot do this thing, but `[an issue exists](LINK)` to change this behavior. - -You can say that we plan to remove a feature. - -#### Legal disclaimer for future features - -If you **must** write about features we have not yet delivered, put this exact disclaimer near the content it applies to. - -```markdown -DISCLAIMER: -This page contains information related to upcoming products, features, and functionality. -It is important to note that the information presented is for informational purposes only. -Please do not rely on this information for purchasing or planning purposes. -As with all projects, the items mentioned on this page are subject to change or delay. -The development, release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. -``` - -It renders on the GitLab documentation site as: - -DISCLAIMER: -This page contains information related to upcoming products, features, and functionality. -It is important to note that the information presented is for informational purposes only. -Please do not rely on this information for purchasing or planning purposes. -As with all projects, the items mentioned on this page are subject to change or delay. -The development, release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. - -If all of the content on the page is not available, use the disclaimer once at the top of the page. - -If the content in a topic is not ready, use the disclaimer in the topic. - -### Removing versions after each major release - -When a major GitLab release occurs, we remove all references -to now-unsupported versions. This removal includes version-specific instructions. For example, -if GitLab version 12.1 and later are supported, -instructions for users of GitLab 11 should be removed. - -[View the list of supported versions](https://about.gitlab.com/support/statement-of-support.html#version-support). - -To view historical information about a feature, review GitLab -[release posts](https://about.gitlab.com/releases/), or search for the issue or -merge request where the work was done. - ## Products and features Refer to the information in this section when describing products and features @@ -1664,7 +1408,7 @@ pricing page. For example: You must assign a tier badge: -- To all H1 topic headings. +- To all H1 topic headings, except the pages under `doc/development/*`. - To topic headings that don't apply to the same tier as the H1. To add a tier badge to a heading, add the relevant tier badge @@ -1692,10 +1436,15 @@ functionality is described. | Only GitLab Premium SaaS and higher tiers (no self-managed instances) | `**(PREMIUM SAAS)**` | | Only GitLab Ultimate SaaS (no self-managed instances) | `**(ULTIMATE SAAS)**` | -Topics that mention the `gitlab.rb` file are referring to -self-managed instances of GitLab. To prevent confusion, include the relevant `TIER SELF` -tier badge on the highest applicable heading level on -the page. +Topics that are only for instance administrators should be badged `<TIER> SELF`. Instance +administrator documentation often includes sections that mention: + +- Changing the `gitlab.rb` or `gitlab.yml` files. +- Accessing the rails console or running Rake tasks. +- Doing things in the Admin Area. + +These pages should also mention if the tasks can only be accomplished by an +instance administrator. ## Specific sections diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index 65f6a0a328b..e7d927de2cf 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -164,7 +164,15 @@ Use lowercase for **boards**, **issue boards**, and **epic boards**. Use **text box** to refer to the UI field. Do not use **field** or **box**. For example: -- In the **Variable name** text box, enter `my text`. +- In the **Variable name** text box, enter a value. + + +## bullet + +Don't refer to individual items in an ordered or unordered list as **bullets**. Use **list item** instead. If you need to be less ambiguous, you can use: + +- **Ordered list item** for items in an ordered list. +- **Unordered list item** for items in an unordered list. ## button @@ -318,7 +326,22 @@ Use **active** or **on** instead. ([Vale](../testing.md#vale) rule: [`InclusionA ## enter -Use **enter** instead of **type** when talking about putting values into text boxes. +In most cases, use **enter** rather than **type**. + +- **Enter** encompasses multiple ways to enter information, including speech and keyboard. +- **Enter** assumes that the user puts a value in a field and then moves the cursor outside the field (or presses <kbd>Enter</kbd>). + **Enter** includes both the entering of the content and the action to validate the content. + +For example: + +- In the **Variable name** text box, enter a value. +- In the **Variable name** text box, enter `my text`. + +When you use **Enter** to refer to the key on a keyboard, use the HTML `<kbd>` tag: + +- To view the list of results, press <kbd>Enter</kbd>. + +See also [**type**](#type). ## epic @@ -356,7 +379,7 @@ Use **box** instead of **field** or **text box**. Use: -- In the **Variable name** box, enter `my text`. +- In the **Variable name** text box, enter `my text`. Instead of: @@ -392,6 +415,13 @@ Do not make **GitLab** possessive (GitLab's). This guidance follows [GitLab Trad **GitLab.com** refers to the GitLab instance managed by GitLab itself. +## GitLab Flavored Markdown + +When possible, spell out [**GitLab Flavored Markdown**](../../../user/markdown.md). +([Vale](../testing.md#vale) rule: [`GLFM.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SubstitutionSuggestions.yml)) + +If you must abbreviate, do not use **GFM**. Use **GLFM** instead. + ## GitLab SaaS **GitLab SaaS** refers to the product license that provides access to GitLab.com. It does not refer to the @@ -518,6 +548,25 @@ Instead of: Do not use **list** when referring to a [**dropdown list**](#dropdown-list). Use the full phrase **dropdown list** instead. +## license + +When writing about licenses: + +- Do not use variations such as **cloud license**, **offline license**, or **legacy license**. +- Do not use interchangeably with **subscription**: + - A license grants users access to the subscription they purchased, and contains information such as the number of seats they purchased and subscription dates. + - A subscription is the subscription tier that the user purchases. + +Use: + + - Add a license to your instance. + - Purchase a subscription. + +Instead of: + + - Buy a license. + - Purchase a license. + ## log in, log on Do not use **log in** or **log on**. Use [sign in](#sign-in) instead. If the user interface has **Log in**, you can use it. @@ -576,6 +625,11 @@ Use lowercase for **merge requests**. If you use **MR** as the acronym, spell it Use lowercase for **milestones**. +## n/a, N/A, not applicable + +When possible, use **not applicable**. Spelling out the phrase helps non-English speaking users and avoids +capitalization inconsistencies. + ## navigate Do not use **navigate**. Use **go** instead. For example: @@ -950,7 +1004,17 @@ Use [**2FA** and **two-factor authentication**](#2fa-two-factor-authentication) ## type -Do not use **type** if you can avoid it. Use **enter** instead. +Use **type** when the cursor remains in the field you're typing in. For example, +in a search dialog, you begin typing and the field populates results. You do not +click out of the field. + +For example: + +- To view all users named Alex, type `Al`. +- To view all labels for the documentation team, type `doc`. +- For a list of quick actions, type `/`. + +See also [**enter**](#enter). ## update @@ -1031,7 +1095,7 @@ Sometimes you might need to use **yet** when writing a task. If you use **yet**, ensure the surrounding phrases are written in present tense, active voice. -[View guidance about how to write about future features](index.md#promising-features-in-future-versions). +[View guidance about how to write about future features](../versions.md#promising-features-in-future-versions). ([Vale](../testing.md#vale) rule: [`CurrentStatus.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/CurrentStatus.yml)) ## you, your, yours diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index 9facb22669b..81e1eca8724 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -23,6 +23,10 @@ in the relevant projects: - <https://gitlab.com/gitlab-org/gitlab-runner/-/blob/main/.gitlab/ci/docs.gitlab-ci.yml> - <https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/gitlab-ci-config/gitlab-com.yml> - <https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/.gitlab-ci.yml> +- <https://gitlab.com/gitlab-org/cloud-native/gitlab-operator/-/blob/master/.gitlab-ci.yml> + +We also run some documentation tests in the GitLab Development Kit project: +<https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/.gitlab/ci/test.gitlab-ci.yml>. ## Run tests locally @@ -144,6 +148,36 @@ synchronized to the other projects. In `omnibus-gitlab`, `gitlab-runner`, and `c is hard coded for specific projects. 1. Create a merge request and submit it to a technical writer for review and merge. +## Update linting images + +Lint tests run in CI/CD pipelines using images from the `gitlab-docs` [container registry](https://gitlab.com/gitlab-org/gitlab-docs/container_registry). + +If a new version of a dependency is released (like a new version of Ruby), we +should update the images to use the newer version. Then, we can update the configuration +files in each of our documentation projects to point to the new image. + +To update the linting images: + +1. In `gitlab-docs`, open a merge request to update `.gitlab-ci.yml` to use the new tooling + version. ([Example MR](https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/2571)) +1. When merged, start a `Build docs.gitlab.com every 4 hours` [scheduled pipeline](https://gitlab.com/gitlab-org/gitlab-docs/-/pipeline_schedules). +1. Go the pipeline you started, and manually run the relevant build-images job, + for example, `image:docs-lint-markdown`. +1. In the job output, get the name of the new image. + ([Example job output](https://gitlab.com/gitlab-org/gitlab-docs/-/jobs/2335033884#L334)) +1. Verify that the new image was added to the container registry. +1. Open merge requests to update each of these configuration files to point to the new image. + In each merge request, include a small doc update to trigger the job that uses the image. + - <https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/docs.gitlab-ci.yml> ([Example MR](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85177)) + - <https://gitlab.com/gitlab-org/gitlab-runner/-/blob/main/.gitlab/ci/test.gitlab-ci.yml> ([Example MR](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3408)) + - <https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/gitlab-ci-config/gitlab-com.yml> ([Example MR](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6037)) + - <https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/.gitlab-ci.yml> ([Example MR](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2511)) + - <https://gitlab.com/gitlab-org/cloud-native/gitlab-operator/-/blob/master/.gitlab-ci.yml> ([Example MR](https://gitlab.com/gitlab-org/cloud-native/gitlab-operator/-/merge_requests/462)) + - <https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/.gitlab/ci/test.gitlab-ci.yml> ([Example MR](https://gitlab.com/gitlab-org/gitlab-development-kit/-/merge_requests/2417)) +1. In each merge request, check the relevant job output to confirm the updated image was + used for the test. ([Example job output](https://gitlab.com/gitlab-org/charts/gitlab/-/jobs/2335470260#L24)) +1. Assign the merge requests to any technical writer to review and merge. + ## Local linters To help adhere to the [documentation style guidelines](styleguide/index.md), and improve the content @@ -173,6 +207,7 @@ markdownlint configuration is found in the following projects: - [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab) - [`charts`](https://gitlab.com/gitlab-org/charts/gitlab) - [`gitlab-development-kit`](https://gitlab.com/gitlab-org/gitlab-development-kit) +- [`gitlab-operator`](https://gitlab.com/gitlab-org/cloud-native/gitlab-operator) This configuration is also used in build pipelines. @@ -311,7 +346,7 @@ To configure Vale in your editor, install one of the following as appropriate: - Visual Studio Code [`errata-ai.vale-server` extension](https://marketplace.visualstudio.com/items?itemName=errata-ai.vale-server). You can configure the plugin to [display only a subset of alerts](#show-subset-of-vale-alerts). - Vim [ALE plugin](https://github.com/dense-analysis/ale). -- Jetbrains IDEs - No plugin exists, but +- JetBrains IDEs - No plugin exists, but [this issue comment](https://github.com/errata-ai/vale-server/issues/39#issuecomment-751714451) contains tips for configuring an external tool. - Emacs [Flycheck extension](https://github.com/flycheck/flycheck). diff --git a/doc/development/documentation/versions.md b/doc/development/documentation/versions.md new file mode 100644 index 00000000000..0f2bdca4c73 --- /dev/null +++ b/doc/development/documentation/versions.md @@ -0,0 +1,232 @@ +--- +info: For assistance with this Style Guide page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-other-projects-and-subjects. +stage: none +group: unassigned +description: 'Writing styles, markup, formatting, and other standards for GitLab Documentation.' +--- + +# Documenting product versions + +The GitLab product documentation includes version-specific information, +including when features were introduced and when they were updated or removed. + +## View older documentation versions + +Previous versions of the documentation are available on `docs.gitlab.com`. +To view a previous version, select the **Versions** button in the top right. + +To view versions that are not available on `docs.gitlab.com`: + +- View the [documentation archives](https://docs.gitlab.com/archives/). +- Go to the GitLab repository and select the version-specific branch. For example, + the [13.2 branch](https://gitlab.com/gitlab-org/gitlab/-/tree/13-2-stable-ee/doc) has the + documentation for GitLab 13.2. + +## Documenting version-specific features + +When a feature is added or updated, you can include its version information +either as a **Version history** bullet or as an inline text reference. + +You do not need to add version information on the pages in the `/development` directory. + +### Add a **Version history** item + +If all content in a topic is related, add a version history item after the topic heading. +For example: + +```markdown +## Feature name + +> [Introduced](<link-to-issue>) in GitLab 11.3. + +This feature does something. +``` + +The item text must include these words in order. Capitalization doesn't matter. + +- `introduced`, `enabled`, `deprecated`, `changed`, `moved`, `recommended`, `removed`, or `renamed` +- `in` or `to` +- `GitLab` + +If possible, include a link to the related issue, merge request, or epic. +Do not link to the pricing page. Do not include the subscription tier. + +#### Introducing a new feature + +If you use `introduced`, start the sentence with the feature name or a gerund: + +```markdown +> - Notifications for expiring tokens [introduced](<link-to-issue>) in GitLab 11.3. +> - Creating an issue from an issue board [introduced](<link-to-issue>) in GitLab 13.1. +``` + +#### Moving subscription tiers + +If a feature is moved to another subscription tier, use `moved`: + +```markdown +> - [Moved](<link-to-issue>) from GitLab Ultimate to GitLab Premium in 11.8. +> - [Moved](<link-to-issue>) from GitLab Premium to GitLab Free in 12.0. +``` + +### Inline version text + +If you're adding content to an existing topic, you can add version information +inline with the existing text. If possible, include a link to the related issue, +merge request, or epic. For example: + +```markdown +The voting strategy [in GitLab 13.4 and later](<link-to-issue>) requires the primary and secondary +voters to agree. +``` + +## Deprecations and removals + +When features are deprecated and removed, update the related documentation. + +API documentation follows these guidelines, but the GraphQL docs use +a [separate process](../api_graphql_styleguide.md#deprecating-schema-items). + +### Deprecate a page or topic + +To deprecate a page or topic: + +1. Add `(deprecated)` after the title. Use a warning to explain when it was deprecated, + when it will be removed, and the replacement feature. + + ```markdown + ## Title (deprecated) **(ULTIMATE SELF)** + + WARNING: + This feature was [deprecated](<link-to-issue>) in GitLab 14.8 + and is planned for removal in 15.4. Use [feature X](<link-to-issue>) instead. + ``` + + If you're not sure when the feature will be removed or no + replacement feature exists, you don't need to add this information. + +1. If the deprecation is a breaking change, add this text: + + ```markdown + This change is a breaking change. + ``` + + You can add any additional context-specific details that might help users. + +1. Open a merge request to add the word `(deprecated)` to the left nav, after the page title. + +### Remove a page + +Mark content as removed during the release the feature was removed. +The title and a removed indicator remains until three months after the removal. + +To remove a page: + +1. Leave the page title. Remove all other content, including the version history items and the word `WARNING:`. +1. After the title, change `(deprecated)` to `(removed)`. +1. Update the YAML metadata: + - For `remove_date`, set the value to a date three months after + the release when the feature was removed. + - For the `redirect_to`, set a path to a file that makes sense. If no obvious + page exists, use the docs home page. + + ```markdown + --- + stage: Enablement + group: Global Search + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments + remove_date: '2022-08-02' + redirect_to: '../newpath/to/file/index.md' + --- + + # Title (removed) **(ULTIMATE SELF)** + + This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8 + and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in 15.0. + Use [feature X](<link-to-issue>) instead. + ``` + +1. Remove the page's entry from the global navigation by editing [`navigation.yaml`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/content/_data/navigation.yaml) in `gitlab-docs`. + +This content is removed from the documentation as part of the Technical Writing team's +[regularly scheduled tasks](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#regularly-scheduled-tasks). + +### Remove a topic + +To remove a topic: + +1. Leave the title and the details of the deprecation and removal. Remove all other content, + including the version history items and the word `WARNING:`. +1. Add `(removed)` after the title. +1. Add the following HTML comments above and below the topic. + For the `remove_date`, set a date three months after the release where it was removed. + + ```markdown + <!--- start_remove The following content will be removed on remove_date: '2023-08-22' --> + + ## Title (removed) **(ULTIMATE SELF)** + + This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8 + and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in 15.0. + Use [feature X](<link-to-issue>) instead. + + <!--- end_remove --> + ``` + +This content is removed from the documentation as part of the Technical Writing team's +[regularly scheduled tasks](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#regularly-scheduled-tasks). + +## Which versions are removed + +GitLab supports the current major version and two previous major versions. +For example, if 14.0 is the current major version, all major and minor releases of +GitLab 14.0, 13.0 and 12.0 are supported. + +[View the list of supported versions](https://about.gitlab.com/support/statement-of-support.html#version-support). + +If you see version history items or inline text that refers to unsupported versions, you can remove it. + +Historical feature information is available in [release posts](https://about.gitlab.com/releases/) +or by searching for the issue or merge request where the work was done. + +## Promising features in future versions + +Do not promise to deliver features in a future release. For example, avoid phrases like, +"Support for this feature is planned." + +We cannot guarantee future feature work, and promises +like these can raise legal issues. Instead, say that an issue exists. +For example: + +- Support for improvements is tracked `[in this issue](LINK)`. +- You cannot do this thing, but `[an issue exists](LINK)` to change this behavior. + +You can say that we plan to remove a feature. + +### Legal disclaimer for future features + +If you **must** write about features we have not yet delivered, put this exact disclaimer near the content it applies to. + +```markdown +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. +``` + +It renders on the GitLab documentation site as: + +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +If all of the content on the page is not available, use the disclaimer once at the top of the page. + +If the content in a topic is not ready, use the disclaimer in the topic. diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index a12af51e436..fb43a2e995a 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -151,7 +151,7 @@ Remember: Ensure the following if skipping an initial Technical Writer review: - [Product badges](styleguide/index.md#product-tier-badges) are applied. -- The GitLab [version](styleguide/index.md#gitlab-versions) that +- The GitLab [version](versions.md) that introduced the feature is included. - Changes to headings don't affect in-app hyperlinks. - Specific [user permissions](../../user/permissions.md) are documented. diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 019dbb13599..28cf6d4e1e3 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -74,6 +74,30 @@ setting the [`FOSS_ONLY` environment variable](https://gitlab.com/gitlab-org/git to something that evaluates as `true`. The same works for running tests (for example `FOSS_ONLY=1 yarn jest`). +### Running feature specs as CE + +When running [feature specs](testing_guide/best_practices.md#system--feature-tests) +as CE, you should ensure that the edition of backend and frontend match. +To do so: + +1. Set the `FOSS_ONLY=1` environment variable: + + ```shell + export FOSS_ONLY=1 + ``` + +1. Start GDK: + + ```shell + gdk start + ``` + +1. Run feature specs: + + ```shell + bin/rspec spec/features/<path_to_your_spec> + ``` + ## CI pipelines in a FOSS context By default, merge request pipelines for development run in an EE-context only. If you are diff --git a/doc/development/emails.md b/doc/development/emails.md index b8e390988bd..a5c2789a3ea 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -86,7 +86,7 @@ See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html# # The IDLE command timeout. idle_timeout: 60 - # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery expunge_deleted: false ``` diff --git a/doc/development/event_store.md b/doc/development/event_store.md index 967272dcf2e..afd5640271e 100644 --- a/doc/development/event_store.md +++ b/doc/development/event_store.md @@ -313,17 +313,17 @@ we have added helpers and shared examples to standardize the way we test subscri ```ruby RSpec.describe MergeRequests::UpdateHeadPipelineWorker do - let(:event) { Ci::PipelineCreatedEvent.new(data: ({ pipeline_id: pipeline.id })) } + let(:pipeline_created_event) { Ci::PipelineCreatedEvent.new(data: ({ pipeline_id: pipeline.id })) } # This shared example ensures that an event is published and correctly processed by # the current subscriber (`described_class`). - it_behaves_like 'consumes the published event' do - let(:event) { event } + it_behaves_like 'subscribes to event' do + let(:event) { pipeline_created_event } end it 'does something' do # This helper directly executes `perform` ensuring that `handle_event` is called correctly. - consume_event(subscriber: described_class, event: event) + consume_event(subscriber: described_class, event: pipeline_created_event) # run expectations end diff --git a/doc/development/experiment_guide/experiment_code_reviews.md b/doc/development/experiment_guide/experiment_code_reviews.md new file mode 100644 index 00000000000..fdde89caa34 --- /dev/null +++ b/doc/development/experiment_guide/experiment_code_reviews.md @@ -0,0 +1,25 @@ +--- +stage: Growth +group: Adoption +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Experiment code reviews + +Experiments' code quality can fail our standards for several reasons. These +reasons can include not being added to the codebase for a long time, or because +of fast iteration to retrieve data. However, having the experiment run (or not +run) shouldn't impact GitLab availability. To avoid or identify issues, +experiments are initially deployed to a small number of users. Regardless, +experiments still need tests. + +Experiments must have corresponding [frontend or feature tests](../testing_guide/index.md) to ensure they +exist in the application. These tests should help prevent the experiment code from +being removed before the [experiment cleanup process](https://about.gitlab.com/handbook/engineering/development/growth/experimentation/#experiment-cleanup-issue) starts. + +If, as a reviewer or maintainer, you find code that would usually fail review +but is acceptable for now, mention your concerns with a note that there's no +need to change the code. The author can then add a comment to this piece of code +and link to the issue that resolves the experiment. The author or reviewer can add a link to this concern in the +experiment rollout issue under the `Experiment Successful Cleanup Concerns` section of the description. +If the experiment is successful and becomes part of the product, any items that appear under this section will be addressed. diff --git a/doc/development/experiment_guide/experiment_rollout.md b/doc/development/experiment_guide/experiment_rollout.md new file mode 100644 index 00000000000..afa32d75221 --- /dev/null +++ b/doc/development/experiment_guide/experiment_rollout.md @@ -0,0 +1,77 @@ +--- +stage: Growth +group: Adoption +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Experiment rollouts and feature flags + +## Experiment rollout issue + +Each experiment should have an [experiment rollout](https://gitlab.com/groups/gitlab-org/-/boards/1352542) issue to track the experiment from rollout through to cleanup and removal. +The rollout issue is similar to a feature flag rollout issue, and is also used to track the status of an experiment. + +When an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). +After the deadline, the issue needs to be resolved and either: + +- It was successful and the experiment becomes the new default. +- It was not successful and all code related to the experiment is removed. + +In either case, an outcome of the experiment should be posted to the issue with the reasoning for the decision. + +## Turn off all experiments + +When there is a case on GitLab.com (SaaS) that necessitates turning off all experiments, we have this control. + +You can toggle experiments on SaaS on and off using the `gitlab_experiment` [feature flag](../feature_flags). + +This can be done via ChatOps: + +- [disable](../feature_flags/controls.md#disabling-feature-flags): `/chatops run feature set gitlab_experiment false` +- [enable](../feature_flags/controls.md#process): `/chatops run feature delete gitlab_experiment` +- This allows the `default_enabled` [value of true in the yml](https://gitlab.com/gitlab-org/gitlab/-/blob/016430f6751b0c34abb24f74608c80a1a8268f20/config/feature_flags/ops/gitlab_experiment.yml#L8) to be honored. + +## Notes on feature flags + +NOTE: +We use the terms "enabled" and "disabled" here, even though it's against our +[documentation style guide recommendations](../documentation/styleguide/word_list.md#enable) +because these are the terms that the feature flag documentation uses. + +You may already be familiar with the concept of feature flags in GitLab, but using +feature flags in experiments is a bit different. While in general terms, a feature flag +is viewed as being either `on` or `off`, this isn't accurate for experiments. + +Generally, `off` means that when we ask if a feature flag is enabled, it will always +return `false`, and `on` means that it will always return `true`. An interim state, +considered `conditional`, also exists. We take advantage of this trinary state of +feature flags. To understand this `conditional` aspect: consider that either of these +settings puts a feature flag into this state: + +- Setting a `percentage_of_actors` of any percent greater than 0%. +- Enabling it for a single user or group. + +Conditional means that it returns `true` in some situations, but not all situations. + +When a feature flag is disabled (meaning the state is `off`), the experiment is +considered _inactive_. You can visualize this in the [decision tree diagram](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment#how-it-works) +as reaching the first `Running?` node, and traversing the negative path. + +When a feature flag is rolled out to a `percentage_of_actors` or similar (meaning the +state is `conditional`) the experiment is considered to be _running_ +where sometimes the control is assigned, and sometimes the candidate is assigned. +We don't refer to this as being enabled, because that's a confusing and overloaded +term here. In the experiment terms, our experiment is _running_, and the feature flag is +`conditional`. + +When a feature flag is enabled (meaning the state is `on`), the candidate will always be +assigned. + +We should try to be consistent with our terms, and so for experiments, we have an +_inactive_ experiment until we set the feature flag to `conditional`. After which, +our experiment is then considered _running_. If you choose to "enable" your feature flag, +you should consider the experiment to be _resolved_, because everyone is assigned +the candidate unless they've opted out of experimentation. + +As of GitLab 13.10, work is being done to improve this process and how we communicate +about it. diff --git a/doc/development/experiment_guide/experimentation.md b/doc/development/experiment_guide/experimentation.md deleted file mode 100644 index 28100564555..00000000000 --- a/doc/development/experiment_guide/experimentation.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'gitlab_experiment.md' -remove_date: '2022-04-13' ---- - -This document was moved to [another location](gitlab_experiment.md). - -<!-- This redirect file can be deleted after <2022-04-13>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/experiment_guide/gitlab_experiment.md b/doc/development/experiment_guide/gitlab_experiment.md index 78e1f84d701..5ddbe9b3de9 100644 --- a/doc/development/experiment_guide/gitlab_experiment.md +++ b/doc/development/experiment_guide/gitlab_experiment.md @@ -1,586 +1,11 @@ --- -stage: Growth -group: Adoption -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-08-05' --- -# Implementing an A/B/n experiment +This document was moved to [another location](index.md). -## Introduction - -Experiments in GitLab are tightly coupled with the concepts provided by -[Feature flags in development of GitLab](../feature_flags/index.md). You're strongly encouraged -to read and understand the [Feature flags in development of GitLab](../feature_flags/index.md) -portion of the documentation before considering running experiments. Experiments add additional -concepts which may seem confusing or advanced without understanding the underpinnings of how GitLab -uses feature flags in development. One concept: experiments can be run with multiple variants, -which are sometimes referred to as A/B/n tests. - -We use the [`gitlab-experiment` gem](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment), -sometimes referred to as GLEX, to run our experiments. The gem exists in a separate repository -so it can be shared across any GitLab property that uses Ruby. You should feel comfortable reading -the documentation on that project if you want to dig into more advanced topics or open issues. Be -aware that the documentation there reflects what's in the main branch and may not be the same as -the version being used within GitLab. - -## Glossary of terms - -To ensure a shared language, you should understand these fundamental terms we use -when communicating about experiments: - -- `experiment`: Any deviation of code paths we want to run at some times, but not others. -- `context`: A consistent experience we provide in an experiment. -- `control`: The default, or "original" code path. -- `candidate`: Defines an experiment with only one code path. -- `variant(s)`: Defines an experiment with multiple code paths. -- `behaviors`: Used to reference all possible code paths of an experiment, including the control. - -## Implementing an experiment - -[Examples](https://gitlab.com/gitlab-org/growth/growth/-/wikis/GLEX-Framework-code-examples) - -Start by generating a feature flag using the `bin/feature-flag` command as you -normally would for a development feature flag, making sure to use `experiment` for -the type. For the sake of documentation let's name our feature flag (and experiment) -"pill_color". - -```shell -bin/feature-flag pill_color -t experiment -``` - -After you generate the desired feature flag, you can immediately implement an -experiment in code. An experiment implementation can be as simple as: - -```ruby -experiment(:pill_color, actor: current_user) do |e| - e.control { 'control' } - e.variant(:red) { 'red' } - e.variant(:blue) { 'blue' } -end -``` - -When this code executes, the experiment is run, a variant is assigned, and (if within a -controller or view) a `window.gl.experiments.pill_color` object will be available in the -client layer, with details like: - -- The assigned variant. -- The context key for client tracking events. - -In addition, when an experiment runs, an event is tracked for -the experiment `:assignment`. We cover more about events, tracking, and -the client layer later. - -In local development, you can make the experiment active by using the feature flag -interface. You can also target specific cases by providing the relevant experiment -to the call to enable the feature flag: - -```ruby -# Enable for everyone -Feature.enable(:pill_color) - -# Get the `experiment` method -- already available in controllers, views, and mailers. -include Gitlab::Experiment::Dsl -# Enable for only the first user -Feature.enable(:pill_color, experiment(:pill_color, actor: User.first)) -``` - -To roll out your experiment feature flag on an environment, run -the following command using ChatOps (which is covered in more depth in the -[Feature flags in development of GitLab](../feature_flags/index.md) documentation). -This command creates a scenario where half of everyone who encounters -the experiment would be assigned the _control_, 25% would be assigned the _red_ -variant, and 25% would be assigned the _blue_ variant: - -```slack -/chatops run feature set pill_color 50 --actors -``` - -For an even distribution in this example, change the command to set it to 66% instead -of 50. - -NOTE: -To immediately stop running an experiment, use the -`/chatops run feature set pill_color false` command. - -WARNING: -We strongly recommend using the `--actors` flag when using the ChatOps commands, -as anything else may give odd behaviors due to how the caching of variant assignment is -handled. - -We can also implement this experiment in a HAML file with HTML wrappings: - -```haml -#cta-interface - - experiment(:pill_color, actor: current_user) do |e| - - e.control do - .pill-button control - - e.variant(:red) do - .pill-button.red red - - e.variant(:blue) do - .pill-button.blue blue -``` - -### The importance of context - -In our previous example experiment, our context (this is an important term) is a hash -that's set to `{ actor: current_user }`. Context must be unique based on how you -want to run your experiment, and should be understood at a lower level. - -It's expected, and recommended, that you use some of these -contexts to simplify reporting: - -- `{ actor: current_user }`: Assigns a variant and is "sticky" to each user - (or "client" if `current_user` is nil) who enters the experiment. -- `{ project: project }`: Assigns a variant and is "sticky" to the project currently - being viewed. If running your experiment is more useful when viewing a project, - rather than when a specific user is viewing any project, consider this approach. -- `{ group: group }`: Similar to the project example, but applies to a wider - scope of projects and users. -- `{ actor: current_user, project: project }`: Assigns a variant and is "sticky" - to the user who is viewing the given project. This creates a different variant - assignment possibility for every project that `current_user` views. Understand this - can create a large cache size if an experiment like this in a highly trafficked part - of the application. -- `{ wday: Time.current.wday }`: Assigns a variant based on the current day of the - week. In this example, it would consistently assign one variant on Friday, and a - potentially different variant on Saturday. - -Context is critical to how you define and report on your experiment. It's usually -the most important aspect of how you choose to implement your experiment, so consider -it carefully, and discuss it with the wider team if needed. Also, take into account -that the context you choose affects our cache size. - -After the above examples, we can state the general case: *given a specific -and consistent context, we can provide a consistent experience and track events for -that experience.* To dive a bit deeper into the implementation details: a context key -is generated from the context that's provided. Use this context key to: - -- Determine the assigned variant. -- Identify events tracked against that context key. - -We can think about this as the experience that we've rendered, which is both dictated -and tracked by the context key. The context key is used to track the interaction and -results of the experience we've rendered to that context key. These concepts are -somewhat abstract and hard to understand initially, but this approach enables us to -communicate about experiments as something that's wider than just user behavior. - -NOTE: -Using `actor:` utilizes cookies if the `current_user` is nil. If you don't need -cookies though - meaning that the exposed functionality would only be visible to -signed in users - `{ user: current_user }` would be just as effective. - -WARNING: -The caching of variant assignment is done by using this context, and so consider -your impact on the cache size when defining your experiment. If you use -`{ time: Time.current }` you would be inflating the cache size every time the -experiment is run. Not only that, your experiment would not be "sticky" and events -wouldn't be resolvable. - -### Advanced experimentation - -There are two ways to implement an experiment: - -1. The simple experiment style described previously. -1. A more advanced style where an experiment class is provided. - -The advanced style is handled by naming convention, and works similar to what you -would expect in Rails. - -To generate a custom experiment class that can override the defaults in -`ApplicationExperiment` use the Rails generator: - -```shell -rails generate gitlab:experiment pill_color control red blue -``` - -This generates an experiment class in `app/experiments/pill_color_experiment.rb` -with the _behaviors_ we've provided to the generator. Here's an example -of how that class would look after migrating our previous example into it: - -```ruby -class PillColorExperiment < ApplicationExperiment - control { 'control' } - variant(:red) { 'red' } - variant(:blue) { 'blue' } -end -``` - -We can now simplify where we run our experiment to the following call, instead of -providing the block we were initially providing, by explicitly calling `run`: - -```ruby -experiment(:pill_color, actor: current_user).run -``` - -The _behaviors_ we defined in our experiment class represent the default -implementation. You can still use the block syntax to override these _behaviors_ -however, so the following would also be valid: - -```ruby -experiment(:pill_color, actor: current_user) do |e| - e.control { '<strong>control</strong>' } -end -``` - -NOTE: -When passing a block to the `experiment` method, it is implicitly invoked as -if `run` has been called. - -#### Segmentation rules - -You can use runtime segmentation rules to, for instance, segment contexts into a specific -variant. The `segment` method is a callback (like `before_action`) and so allows providing -a block or method name. - -In this example, any user named `'Richard'` would always be assigned the _red_ -variant, and any account older than 2 weeks old would be assigned the _blue_ variant: - -```ruby -class PillColorExperiment < ApplicationExperiment - # ...registered behaviors - - segment(variant: :red) { context.actor.first_name == 'Richard' } - segment :old_account?, variant: :blue - - private - - def old_account? - context.actor.created_at < 2.weeks.ago - end -end -``` - -When an experiment runs, the segmentation rules are executed in the order they're -defined. The first segmentation rule to produce a truthy result assigns the variant. - -In our example, any user named `'Richard'`, regardless of account age, will always -be assigned the _red_ variant. If you want the opposite logic, flip the order. - -NOTE: -Keep in mind when defining segmentation rules: after a truthy result, the remaining -segmentation rules are skipped to achieve optimal performance. - -#### Exclusion rules - -Exclusion rules are similar to segmentation rules, but are intended to determine -if a context should even be considered as something we should include in the experiment -and track events toward. Exclusion means we don't care about the events in relation -to the given context. - -These examples exclude all users named `'Richard'`, *and* any account -older than 2 weeks old. Not only are they given the control behavior - which could -be nothing - but no events are tracked in these cases as well. - -```ruby -class PillColorExperiment < ApplicationExperiment - # ...registered behaviors - - exclude :old_account?, ->{ context.actor.first_name == 'Richard' } - - private - - def old_account? - context.actor.created_at < 2.weeks.ago - end -end -``` - -You may also need to check exclusion in custom tracking logic by calling `should_track?`: - -```ruby -class PillColorExperiment < ApplicationExperiment - # ...registered behaviors - - def expensive_tracking_logic - return unless should_track? - - track(:my_event, value: expensive_method_call) - end -end -``` - -### Tracking events - -One of the most important aspects of experiments is gathering data and reporting on -it. You can use the `track` method to track events across an experimental implementation. -You can track events consistently to an experiment if you provide the same context between -calls to your experiment. If you do not yet understand context, you should read -about contexts now. - -We can assume we run the experiment in one or a few places, but -track events potentially in many places. The tracking call remains the same, with -the arguments you would normally use when -[tracking events using snowplow](../snowplow/index.md). The easiest example -of tracking an event in Ruby would be: - -```ruby -experiment(:pill_color, actor: current_user).track(:clicked) -``` - -When you run an experiment with any of the examples so far, an `:assignment` event -is tracked automatically by default. All events that are tracked from an -experiment have a special -[experiment context](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/1-0-3) -added to the event. This can be used - typically by the data team - to create a connection -between the events on a given experiment. - -If our current user hasn't encountered the experiment yet (meaning where the experiment -is run), and we track an event for them, they are assigned a variant and see -that variant if they ever encountered the experiment later, when an `:assignment` -event would be tracked at that time for them. - -NOTE: -GitLab tries to be sensitive and respectful of our customers regarding tracking, -so our experimentation library allows us to implement an experiment without ever tracking identifying -IDs. It's not always possible, though, based on experiment reporting requirements. -You may be asked from time to time to track a specific record ID in experiments. -The approach is largely up to the PM and engineer creating the implementation. -No recommendations are provided here at this time. - -## Testing with RSpec - -In the course of working with experiments, you'll probably want to utilize the RSpec -tooling that's built in. This happens automatically for files in `spec/experiments`, but -for other files and specs you want to include it in, you can specify the `:experiment` type: - -```ruby -it "tests experiments nicely", :experiment do -end -``` - -### Stub helpers - -You can stub experiments using `stub_experiments`. Pass it a hash using experiment -names as the keys, and the variants you want each to resolve to, as the values: - -```ruby -# Ensures the experiments named `:example` & `:example2` are both "enabled" and -# that each will resolve to the given variant (`:my_variant` and `:control` -# respectively). -stub_experiments(example: :my_variant, example2: :control) - -experiment(:example) do |e| - e.enabled? # => true - e.assigned.name # => 'my_variant' -end - -experiment(:example2) do |e| - e.enabled? # => true - e.assigned.name # => 'control' -end -``` - -### Exclusion, segmentation, and behavior matchers - -You can also test things like the registered behaviors, the exclusions, and -segmentations using the matchers. - -```ruby -class ExampleExperiment < ApplicationExperiment - control { } - candidate { '_candidate_' } - - exclude { context.actor.first_name == 'Richard' } - segment(variant: :candidate) { context.actor.username == 'jejacks0n' } -end - -excluded = double(username: 'rdiggitty', first_name: 'Richard') -segmented = double(username: 'jejacks0n', first_name: 'Jeremy') - -# register_behavior matcher -expect(experiment(:example)).to register_behavior(:control) -expect(experiment(:example)).to register_behavior(:candidate).with('_candidate_') - -# exclude matcher -expect(experiment(:example)).to exclude(actor: excluded) -expect(experiment(:example)).not_to exclude(actor: segmented) - -# segment matcher -expect(experiment(:example)).to segment(actor: segmented).into(:candidate) -expect(experiment(:example)).not_to segment(actor: excluded) -``` - -### Tracking matcher - -Tracking events is a major aspect of experimentation. We try -to provide a flexible way to ensure your tracking calls are covered. - -You can do this on the instance level or at an "any instance" level: - -```ruby -subject = experiment(:example) - -expect(subject).to track(:my_event) - -subject.track(:my_event) -``` - -You can use the `on_next_instance` chain method to specify that it will happen -on the next instance of the experiment. This helps you if you're calling -`experiment(:example).track` downstream: - -```ruby -expect(experiment(:example)).to track(:my_event).on_next_instance - -experiment(:example).track(:my_event) -``` - -A full example of the methods you can chain onto the `track` matcher: - -```ruby -expect(experiment(:example)).to track(:my_event, value: 1, property: '_property_') - .on_next_instance - .with_context(foo: :bar) - .for(:variant_name) - -experiment(:example, :variant_name, foo: :bar).track(:my_event, value: 1, property: '_property_') -``` - -## Experiments in the client layer - -Any experiment that's been run in the request lifecycle surfaces in `window.gl.experiments`, -and matches [this schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/1-0-3) -so it can be used when resolving experimentation in the client layer. - -Given that we've defined a class for our experiment, and have defined the variants for it, we can publish that experiment in a couple ways. - -The first way is simply by running the experiment. Assuming the experiment has been run, it will surface in the client layer without having to do anything special. - -The second way doesn't run the experiment and is intended to be used if the experiment only needs to surface in the client layer. To accomplish this we can simply `.publish` the experiment. This won't run any logic, but does surface the experiment details in the client layer so they can be utilized there. - -An example might be to publish an experiment in a `before_action` in a controller. Assuming we've defined the `PillColorExperiment` class, like we have above, we can surface it to the client by publishing it instead of running it: - -```ruby -before_action -> { experiment(:pill_color).publish }, only: [:show] -``` - -You can then see this surface in the JavaScript console: - -```javascript -window.gl.experiments // => { pill_color: { excluded: false, experiment: "pill_color", key: "ca63ac02", variant: "candidate" } } -``` - -### Using experiments in Vue - -With the `gitlab-experiment` component, you can define slots that match the name of the -variants pushed to `window.gl.experiments`. - -We can make use of the named slots in the Vue component, that match the behaviors defined in : - -```vue -<script> -import GitlabExperiment from '~/experimentation/components/gitlab_experiment.vue'; - -export default { - components: { GitlabExperiment } -} -</script> - -<template> - <gitlab-experiment name="pill_color"> - <template #control> - <button class="bg-default">Click default button</button> - </template> - - <template #red> - <button class="bg-red">Click red button</button> - </template> - - <template #blue> - <button class="bg-blue">Click blue button</button> - </template> - </gitlab-experiment> -</template> -``` - -NOTE: -When there is no experiment data in the `window.gl.experiments` object for the given experiment name, the `control` slot will be used, if it exists. - -## Test with Jest - -### Stub Helpers - -You can stub experiments using the `stubExperiments` helper defined in `spec/frontend/__helpers__/experimentation_helper.js`. - -```javascript -import { stubExperiments } from 'helpers/experimentation_helper'; -import { getExperimentData } from '~/experimentation/utils'; - -describe('when my_experiment is enabled', () => { - beforeEach(() => { - stubExperiments({ my_experiment: 'candidate' }); - }); - - it('sets the correct data', () => { - expect(getExperimentData('my_experiment')).toEqual({ experiment: 'my_experiment', variant: 'candidate' }); - }); -}); -``` - -NOTE: -This method of stubbing in Jest specs will not automatically un-stub itself at the end of the test. We merge our stubbed experiment in with all the other global data in `window.gl`. If you need to remove the stubbed experiment(s) after your test or ensure a clean global object before your test, you'll need to manage the global object directly yourself: - -```javascript -describe('tests that care about global state', () => { - const originalObjects = []; - - beforeEach(() => { - // For backwards compatibility for now, we're using both window.gon & window.gl - originalObjects.push(window.gon, window.gl); - }); - - afterEach(() => { - [window.gon, window.gl] = originalObjects; - }); - - it('stubs experiment in fresh global state', () => { - stubExperiment({ my_experiment: 'candidate' }); - // ... - }); -}) -``` - -## Notes on feature flags - -NOTE: -We use the terms "enabled" and "disabled" here, even though it's against our -[documentation style guide recommendations](../documentation/styleguide/word_list.md#enable) -because these are the terms that the feature flag documentation uses. - -You may already be familiar with the concept of feature flags in GitLab, but using -feature flags in experiments is a bit different. While in general terms, a feature flag -is viewed as being either `on` or `off`, this isn't accurate for experiments. - -Generally, `off` means that when we ask if a feature flag is enabled, it will always -return `false`, and `on` means that it will always return `true`. An interim state, -considered `conditional`, also exists. We take advantage of this trinary state of -feature flags. To understand this `conditional` aspect: consider that either of these -settings puts a feature flag into this state: - -- Setting a `percentage_of_actors` of any percent greater than 0%. -- Enabling it for a single user or group. - -Conditional means that it returns `true` in some situations, but not all situations. - -When a feature flag is disabled (meaning the state is `off`), the experiment is -considered _inactive_. You can visualize this in the [decision tree diagram](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment#how-it-works) -as reaching the first `Running?` node, and traversing the negative path. - -When a feature flag is rolled out to a `percentage_of_actors` or similar (meaning the -state is `conditional`) the experiment is considered to be _running_ -where sometimes the control is assigned, and sometimes the candidate is assigned. -We don't refer to this as being enabled, because that's a confusing and overloaded -term here. In the experiment terms, our experiment is _running_, and the feature flag is -`conditional`. - -When a feature flag is enabled (meaning the state is `on`), the candidate will always be -assigned. - -We should try to be consistent with our terms, and so for experiments, we have an -_inactive_ experiment until we set the feature flag to `conditional`. After which, -our experiment is then considered _running_. If you choose to "enable" your feature flag, -you should consider the experiment to be _resolved_, because everyone is assigned -the candidate unless they've opted out of experimentation. - -As of GitLab 13.10, work is being done to improve this process and how we communicate -about it. +<!-- This redirect file can be deleted after 2022-08-05. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/experiment_guide/implementing_experiments.md b/doc/development/experiment_guide/implementing_experiments.md new file mode 100644 index 00000000000..3c33d015108 --- /dev/null +++ b/doc/development/experiment_guide/implementing_experiments.md @@ -0,0 +1,369 @@ +--- +stage: Growth +group: Adoption +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Implementing an A/B/n experiment + +## Implementing an experiment + +[Examples](https://gitlab.com/gitlab-org/growth/growth/-/wikis/GLEX-Framework-code-examples) + +Start by generating a feature flag using the `bin/feature-flag` command as you +normally would for a development feature flag, making sure to use `experiment` for +the type. For the sake of documentation let's name our feature flag (and experiment) +"pill_color". + +```shell +bin/feature-flag pill_color -t experiment +``` + +After you generate the desired feature flag, you can immediately implement an +experiment in code. An experiment implementation can be as simple as: + +```ruby +experiment(:pill_color, actor: current_user) do |e| + e.control { 'control' } + e.variant(:red) { 'red' } + e.variant(:blue) { 'blue' } +end +``` + +When this code executes, the experiment is run, a variant is assigned, and (if within a +controller or view) a `window.gl.experiments.pill_color` object will be available in the +client layer, with details like: + +- The assigned variant. +- The context key for client tracking events. + +In addition, when an experiment runs, an event is tracked for +the experiment `:assignment`. We cover more about events, tracking, and +the client layer later. + +In local development, you can make the experiment active by using the feature flag +interface. You can also target specific cases by providing the relevant experiment +to the call to enable the feature flag: + +```ruby +# Enable for everyone +Feature.enable(:pill_color) + +# Get the `experiment` method -- already available in controllers, views, and mailers. +include Gitlab::Experiment::Dsl +# Enable for only the first user +Feature.enable(:pill_color, experiment(:pill_color, actor: User.first)) +``` + +To roll out your experiment feature flag on an environment, run +the following command using ChatOps (which is covered in more depth in the +[Feature flags in development of GitLab](../feature_flags/index.md) documentation). +This command creates a scenario where half of everyone who encounters +the experiment would be assigned the _control_, 25% would be assigned the _red_ +variant, and 25% would be assigned the _blue_ variant: + +```slack +/chatops run feature set pill_color 50 --actors +``` + +For an even distribution in this example, change the command to set it to 66% instead +of 50. + +NOTE: +To immediately stop running an experiment, use the +`/chatops run feature set pill_color false` command. + +WARNING: +We strongly recommend using the `--actors` flag when using the ChatOps commands, +as anything else may give odd behaviors due to how the caching of variant assignment is +handled. + +We can also implement this experiment in a HAML file with HTML wrappings: + +```haml +#cta-interface + - experiment(:pill_color, actor: current_user) do |e| + - e.control do + .pill-button control + - e.variant(:red) do + .pill-button.red red + - e.variant(:blue) do + .pill-button.blue blue +``` + +### The importance of context + +In our previous example experiment, our context (this is an important term) is a hash +that's set to `{ actor: current_user }`. Context must be unique based on how you +want to run your experiment, and should be understood at a lower level. + +It's expected, and recommended, that you use some of these +contexts to simplify reporting: + +- `{ actor: current_user }`: Assigns a variant and is "sticky" to each user + (or "client" if `current_user` is nil) who enters the experiment. +- `{ project: project }`: Assigns a variant and is "sticky" to the project currently + being viewed. If running your experiment is more useful when viewing a project, + rather than when a specific user is viewing any project, consider this approach. +- `{ group: group }`: Similar to the project example, but applies to a wider + scope of projects and users. +- `{ actor: current_user, project: project }`: Assigns a variant and is "sticky" + to the user who is viewing the given project. This creates a different variant + assignment possibility for every project that `current_user` views. Understand this + can create a large cache size if an experiment like this in a highly trafficked part + of the application. +- `{ wday: Time.current.wday }`: Assigns a variant based on the current day of the + week. In this example, it would consistently assign one variant on Friday, and a + potentially different variant on Saturday. + +Context is critical to how you define and report on your experiment. It's usually +the most important aspect of how you choose to implement your experiment, so consider +it carefully, and discuss it with the wider team if needed. Also, take into account +that the context you choose affects our cache size. + +After the above examples, we can state the general case: *given a specific +and consistent context, we can provide a consistent experience and track events for +that experience.* To dive a bit deeper into the implementation details: a context key +is generated from the context that's provided. Use this context key to: + +- Determine the assigned variant. +- Identify events tracked against that context key. + +We can think about this as the experience that we've rendered, which is both dictated +and tracked by the context key. The context key is used to track the interaction and +results of the experience we've rendered to that context key. These concepts are +somewhat abstract and hard to understand initially, but this approach enables us to +communicate about experiments as something that's wider than just user behavior. + +NOTE: +Using `actor:` utilizes cookies if the `current_user` is nil. If you don't need +cookies though - meaning that the exposed functionality would only be visible to +signed in users - `{ user: current_user }` would be just as effective. + +WARNING: +The caching of variant assignment is done by using this context, and so consider +your impact on the cache size when defining your experiment. If you use +`{ time: Time.current }` you would be inflating the cache size every time the +experiment is run. Not only that, your experiment would not be "sticky" and events +wouldn't be resolvable. + +### Advanced experimentation + +There are two ways to implement an experiment: + +1. The simple experiment style described previously. +1. A more advanced style where an experiment class is provided. + +The advanced style is handled by naming convention, and works similar to what you +would expect in Rails. + +To generate a custom experiment class that can override the defaults in +`ApplicationExperiment` use the Rails generator: + +```shell +rails generate gitlab:experiment pill_color control red blue +``` + +This generates an experiment class in `app/experiments/pill_color_experiment.rb` +with the _behaviors_ we've provided to the generator. Here's an example +of how that class would look after migrating our previous example into it: + +```ruby +class PillColorExperiment < ApplicationExperiment + control { 'control' } + variant(:red) { 'red' } + variant(:blue) { 'blue' } +end +``` + +We can now simplify where we run our experiment to the following call, instead of +providing the block we were initially providing, by explicitly calling `run`: + +```ruby +experiment(:pill_color, actor: current_user).run +``` + +The _behaviors_ we defined in our experiment class represent the default +implementation. You can still use the block syntax to override these _behaviors_ +however, so the following would also be valid: + +```ruby +experiment(:pill_color, actor: current_user) do |e| + e.control { '<strong>control</strong>' } +end +``` + +NOTE: +When passing a block to the `experiment` method, it is implicitly invoked as +if `run` has been called. + +#### Segmentation rules + +You can use runtime segmentation rules to, for instance, segment contexts into a specific +variant. The `segment` method is a callback (like `before_action`) and so allows providing +a block or method name. + +In this example, any user named `'Richard'` would always be assigned the _red_ +variant, and any account older than 2 weeks old would be assigned the _blue_ variant: + +```ruby +class PillColorExperiment < ApplicationExperiment + # ...registered behaviors + + segment(variant: :red) { context.actor.first_name == 'Richard' } + segment :old_account?, variant: :blue + + private + + def old_account? + context.actor.created_at < 2.weeks.ago + end +end +``` + +When an experiment runs, the segmentation rules are executed in the order they're +defined. The first segmentation rule to produce a truthy result assigns the variant. + +In our example, any user named `'Richard'`, regardless of account age, will always +be assigned the _red_ variant. If you want the opposite logic, flip the order. + +NOTE: +Keep in mind when defining segmentation rules: after a truthy result, the remaining +segmentation rules are skipped to achieve optimal performance. + +#### Exclusion rules + +Exclusion rules are similar to segmentation rules, but are intended to determine +if a context should even be considered as something we should include in the experiment +and track events toward. Exclusion means we don't care about the events in relation +to the given context. + +These examples exclude all users named `'Richard'`, *and* any account +older than 2 weeks old. Not only are they given the control behavior - which could +be nothing - but no events are tracked in these cases as well. + +```ruby +class PillColorExperiment < ApplicationExperiment + # ...registered behaviors + + exclude :old_account?, ->{ context.actor.first_name == 'Richard' } + + private + + def old_account? + context.actor.created_at < 2.weeks.ago + end +end +``` + +You may also need to check exclusion in custom tracking logic by calling `should_track?`: + +```ruby +class PillColorExperiment < ApplicationExperiment + # ...registered behaviors + + def expensive_tracking_logic + return unless should_track? + + track(:my_event, value: expensive_method_call) + end +end +``` + +### Tracking events + +One of the most important aspects of experiments is gathering data and reporting on +it. You can use the `track` method to track events across an experimental implementation. +You can track events consistently to an experiment if you provide the same context between +calls to your experiment. If you do not yet understand context, you should read +about contexts now. + +We can assume we run the experiment in one or a few places, but +track events potentially in many places. The tracking call remains the same, with +the arguments you would normally use when +[tracking events using snowplow](../snowplow/index.md). The easiest example +of tracking an event in Ruby would be: + +```ruby +experiment(:pill_color, actor: current_user).track(:clicked) +``` + +When you run an experiment with any of the examples so far, an `:assignment` event +is tracked automatically by default. All events that are tracked from an +experiment have a special +[experiment context](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/1-0-3) +added to the event. This can be used - typically by the data team - to create a connection +between the events on a given experiment. + +If our current user hasn't encountered the experiment yet (meaning where the experiment +is run), and we track an event for them, they are assigned a variant and see +that variant if they ever encountered the experiment later, when an `:assignment` +event would be tracked at that time for them. + +NOTE: +GitLab tries to be sensitive and respectful of our customers regarding tracking, +so our experimentation library allows us to implement an experiment without ever tracking identifying +IDs. It's not always possible, though, based on experiment reporting requirements. +You may be asked from time to time to track a specific record ID in experiments. +The approach is largely up to the PM and engineer creating the implementation. +No recommendations are provided here at this time. + +## Experiments in the client layer + +Any experiment that's been run in the request lifecycle surfaces in `window.gl.experiments`, +and matches [this schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_experiment/jsonschema/1-0-3) +so it can be used when resolving experimentation in the client layer. + +Given that we've defined a class for our experiment, and have defined the variants for it, we can publish that experiment in a couple ways. + +The first way is simply by running the experiment. Assuming the experiment has been run, it will surface in the client layer without having to do anything special. + +The second way doesn't run the experiment and is intended to be used if the experiment only needs to surface in the client layer. To accomplish this we can simply `.publish` the experiment. This won't run any logic, but does surface the experiment details in the client layer so they can be utilized there. + +An example might be to publish an experiment in a `before_action` in a controller. Assuming we've defined the `PillColorExperiment` class, like we have above, we can surface it to the client by publishing it instead of running it: + +```ruby +before_action -> { experiment(:pill_color).publish }, only: [:show] +``` + +You can then see this surface in the JavaScript console: + +```javascript +window.gl.experiments // => { pill_color: { excluded: false, experiment: "pill_color", key: "ca63ac02", variant: "candidate" } } +``` + +### Using experiments in Vue + +With the `gitlab-experiment` component, you can define slots that match the name of the +variants pushed to `window.gl.experiments`. + +We can make use of the named slots in the Vue component, that match the behaviors defined in : + +```vue +<script> +import GitlabExperiment from '~/experimentation/components/gitlab_experiment.vue'; + +export default { + components: { GitlabExperiment } +} +</script> + +<template> + <gitlab-experiment name="pill_color"> + <template #control> + <button class="bg-default">Click default button</button> + </template> + + <template #red> + <button class="bg-red">Click red button</button> + </template> + + <template #blue> + <button class="bg-blue">Click blue button</button> + </template> + </gitlab-experiment> +</template> +``` + +NOTE: +When there is no experiment data in the `window.gl.experiments` object for the given experiment name, the `control` slot will be used, if it exists. diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index f7af1113b6e..b140cce34fc 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -6,47 +6,46 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Experiment Guide -Experiments can be conducted by any GitLab team, most often the teams from the [Growth Sub-department](https://about.gitlab.com/handbook/engineering/development/growth/). Experiments are not tied to releases because they primarily target GitLab.com. - -Experiments are run as an A/B/n test, and are behind an [experiment feature flag](../feature_flags/#experiment-type) to turn the test on or off. Based on the data the experiment generates, the team decides if the experiment had a positive impact and should be made the new default, or rolled back. - -## Experiment rollout issue - -Each experiment should have an [experiment rollout](https://gitlab.com/groups/gitlab-org/-/boards/1352542) issue to track the experiment from rollout through to cleanup and removal. -The rollout issue is similar to a feature flag rollout issue, and is also used to track the status of an experiment. -When an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). -After the deadline, the issue needs to be resolved and either: - -- It was successful and the experiment becomes the new default. -- It was not successful and all code related to the experiment is removed. - -In either case, an outcome of the experiment should be posted to the issue with the reasoning for the decision. - -## Code reviews - -Experiments' code quality can fail our standards for several reasons. These -reasons can include not being added to the codebase for a long time, or because -of fast iteration to retrieve data. However, having the experiment run (or not -run) shouldn't impact GitLab availability. To avoid or identify issues, -experiments are initially deployed to a small number of users. Regardless, -experiments still need tests. - -Experiments must have corresponding [frontend or feature tests](../testing_guide/index.md) to ensure they -exist in the application. These tests should help prevent the experiment code from -being removed before the [experiment cleanup process](https://about.gitlab.com/handbook/engineering/development/growth/experimentation/#experiment-cleanup-issue) starts. - -If, as a reviewer or maintainer, you find code that would usually fail review -but is acceptable for now, mention your concerns with a note that there's no -need to change the code. The author can then add a comment to this piece of code -and link to the issue that resolves the experiment. The author or reviewer can add a link to this concern in the -experiment rollout issue under the `Experiment Successful Cleanup Concerns` section of the description. -If the experiment is successful and becomes part of the product, any items that appear under this section will be addressed. +Experiments can be conducted by any GitLab team, most often the teams from the +[Growth Sub-department](https://about.gitlab.com/handbook/engineering/development/growth/). +Experiments are not tied to releases because they primarily target GitLab.com. + +Experiments are run as an A/B/n test, and are behind an [experiment feature flag](../feature_flags/#experiment-type) +to turn the test on or off. Based on the data the experiment generates, the team decides +if the experiment had a positive impact and should be made the new default, or rolled back. + +Experiments in GitLab are tightly coupled with the concepts provided by +[Feature flags in development of GitLab](../feature_flags/index.md). You're strongly encouraged +to read and understand the [Feature flags in development of GitLab](../feature_flags/index.md) +portion of the documentation before considering running experiments. Experiments add additional +concepts which may seem confusing or advanced without understanding the underpinnings of how GitLab +uses feature flags in development. One concept: experiments can be run with multiple variants, +which are sometimes referred to as A/B/n tests. + +We use the [`gitlab-experiment` gem](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment), +sometimes referred to as GLEX, to run our experiments. The gem exists in a separate repository +so it can be shared across any GitLab property that uses Ruby. You should feel comfortable reading +the documentation on that project if you want to dig into more advanced topics or open issues. Be +aware that the documentation there reflects what's in the main branch and may not be the same as +the version being used within GitLab. + +## Glossary of terms + +To ensure a shared language, you should understand these fundamental terms we use +when communicating about experiments: + +- `experiment`: Any deviation of code paths we want to run at some times, but not others. +- `context`: A consistent experience we provide in an experiment. +- `control`: The default, or "original" code path. +- `candidate`: Defines an experiment with only one code path. +- `variant(s)`: Defines an experiment with multiple code paths. +- `behaviors`: Used to reference all possible code paths of an experiment, including the control. ## Implementing an experiment [`GLEX`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment) - or `Gitlab::Experiment`, the `gitlab-experiment` gem - is the preferred option for implementing an experiment in GitLab. -For more information, see [Implementing an A/B/n experiment using GLEX](gitlab_experiment.md). +For more information, see [Implementing an A/B/n experiment using GLEX](implementing_experiments.md). This uses [experiment](../feature_flags/index.md#experiment-type) feature flags. @@ -64,15 +63,3 @@ We recommend the following workflow: 1. **If the experiment is a success**, designers add the new icon or illustration to the Pajamas UI kit as part of the cleanup process. Engineers can then add it to the [SVG library](https://gitlab-org.gitlab.io/gitlab-svgs/) and modify the implementation based on the [Frontend Development Guidelines](../fe_guide/icons.md#usage-in-hamlrails-2). - -## Turn off all experiments - -When there is a case on GitLab.com (SaaS) that necessitates turning off all experiments, we have this control. - -You can toggle experiments on SaaS on and off using the `gitlab_experiment` [feature flag](../feature_flags). - -This can be done via chatops: - -- [disable](../feature_flags/controls.md#disabling-feature-flags): `/chatops run feature set gitlab_experiment false` -- [enable](../feature_flags/controls.md#process): `/chatops run feature delete gitlab_experiment` - - This allows the `default_enabled` [value of true in the yml](https://gitlab.com/gitlab-org/gitlab/-/blob/016430f6751b0c34abb24f74608c80a1a8268f20/config/feature_flags/ops/gitlab_experiment.yml#L8) to be honored. diff --git a/doc/development/experiment_guide/testing_experiments.md b/doc/development/experiment_guide/testing_experiments.md new file mode 100644 index 00000000000..08ff91a3deb --- /dev/null +++ b/doc/development/experiment_guide/testing_experiments.md @@ -0,0 +1,150 @@ +--- +stage: Growth +group: Activation +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Testing experiments + +## Testing experiments with RSpec + +In the course of working with experiments, you'll probably want to utilize the RSpec +tooling that's built in. This happens automatically for files in `spec/experiments`, but +for other files and specs you want to include it in, you can specify the `:experiment` type: + +```ruby +it "tests experiments nicely", :experiment do +end +``` + +### Stub helpers + +You can stub experiments using `stub_experiments`. Pass it a hash using experiment +names as the keys, and the variants you want each to resolve to, as the values: + +```ruby +# Ensures the experiments named `:example` & `:example2` are both "enabled" and +# that each will resolve to the given variant (`:my_variant` and `:control` +# respectively). +stub_experiments(example: :my_variant, example2: :control) + +experiment(:example) do |e| + e.enabled? # => true + e.assigned.name # => 'my_variant' +end + +experiment(:example2) do |e| + e.enabled? # => true + e.assigned.name # => 'control' +end +``` + +### Exclusion, segmentation, and behavior matchers + +You can also test things like the registered behaviors, the exclusions, and +segmentations using the matchers. + +```ruby +class ExampleExperiment < ApplicationExperiment + control { } + candidate { '_candidate_' } + + exclude { context.actor.first_name == 'Richard' } + segment(variant: :candidate) { context.actor.username == 'jejacks0n' } +end + +excluded = double(username: 'rdiggitty', first_name: 'Richard') +segmented = double(username: 'jejacks0n', first_name: 'Jeremy') + +# register_behavior matcher +expect(experiment(:example)).to register_behavior(:control) +expect(experiment(:example)).to register_behavior(:candidate).with('_candidate_') + +# exclude matcher +expect(experiment(:example)).to exclude(actor: excluded) +expect(experiment(:example)).not_to exclude(actor: segmented) + +# segment matcher +expect(experiment(:example)).to segment(actor: segmented).into(:candidate) +expect(experiment(:example)).not_to segment(actor: excluded) +``` + +### Tracking matcher + +Tracking events is a major aspect of experimentation. We try +to provide a flexible way to ensure your tracking calls are covered. + +You can do this on the instance level or at an "any instance" level: + +```ruby +subject = experiment(:example) + +expect(subject).to track(:my_event) + +subject.track(:my_event) +``` + +You can use the `on_next_instance` chain method to specify that it will happen +on the next instance of the experiment. This helps you if you're calling +`experiment(:example).track` downstream: + +```ruby +expect(experiment(:example)).to track(:my_event).on_next_instance + +experiment(:example).track(:my_event) +``` + +A full example of the methods you can chain onto the `track` matcher: + +```ruby +expect(experiment(:example)).to track(:my_event, value: 1, property: '_property_') + .on_next_instance + .with_context(foo: :bar) + .for(:variant_name) + +experiment(:example, :variant_name, foo: :bar).track(:my_event, value: 1, property: '_property_') +``` + +## Test with Jest + +### Stub Helpers + +You can stub experiments using the `stubExperiments` helper defined in `spec/frontend/__helpers__/experimentation_helper.js`. + +```javascript +import { stubExperiments } from 'helpers/experimentation_helper'; +import { getExperimentData } from '~/experimentation/utils'; + +describe('when my_experiment is enabled', () => { + beforeEach(() => { + stubExperiments({ my_experiment: 'candidate' }); + }); + + it('sets the correct data', () => { + expect(getExperimentData('my_experiment')).toEqual({ experiment: 'my_experiment', variant: 'candidate' }); + }); +}); +``` + +NOTE: +This method of stubbing in Jest specs will not automatically un-stub itself at the end of the test. We merge our stubbed experiment in with all the other global data in `window.gl`. If you need to remove the stubbed experiments after your test or ensure a clean global object before your test, you'll need to manage the global object directly yourself: + +```javascript +describe('tests that care about global state', () => { + const originalObjects = []; + + beforeEach(() => { + // For backwards compatibility for now, we're using both window.gon & window.gl + originalObjects.push(window.gon, window.gl); + }); + + afterEach(() => { + [window.gon, window.gl] = originalObjects; + }); + + it('stubs experiment in fresh global state', () => { + stubExperiment({ my_experiment: 'candidate' }); + // ... + }); +}) +``` diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index 2e64f52651e..d4c29cb8a24 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -47,7 +47,7 @@ The Content Editor requires two properties: - `renderMarkdown` is an asynchronous function that returns the response (String) of invoking the [Markdown API](../../api/markdown.md). -- `uploadsPath` is a URL that points to a [GitLab upload service](../uploads/implementation.md#upload-encodings) +- `uploadsPath` is a URL that points to a [GitLab upload service](../uploads/index.md) with `multipart/form-data` support. See the [`WikiForm.vue`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/pages/shared/wikis/components/wiki_form.vue#L207) diff --git a/doc/development/fe_guide/design_anti_patterns.md b/doc/development/fe_guide/design_anti_patterns.md index 76825d6ff18..b7238bb2813 100644 --- a/doc/development/fe_guide/design_anti_patterns.md +++ b/doc/development/fe_guide/design_anti_patterns.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Anti-patterns may seem like good approaches at first, but it has been shown that they bring more ills than benefits. These should generally be avoided. -Throughout the GitLab codebase, there may be historic uses of these anti-patterns. Please [use discretion](https://about.gitlab.com/handbook/engineering/principles/#balance-refactoring-and-velocity) +Throughout the GitLab codebase, there may be historic uses of these anti-patterns. Please [use discretion](https://about.gitlab.com/handbook/engineering/development/principles/#balance-refactoring-and-velocity) when figuring out whether or not to refactor, when touching code that uses one of these legacy patterns. NOTE: diff --git a/doc/development/fe_guide/development_process.md b/doc/development/fe_guide/development_process.md index 9921b851344..b4893fd4ef9 100644 --- a/doc/development/fe_guide/development_process.md +++ b/doc/development/fe_guide/development_process.md @@ -103,7 +103,7 @@ With the purpose of being [respectful of others' time](https://about.gitlab.com/ - Before assigning to a maintainer, assign to a reviewer. - If you assigned a merge request or pinged someone directly, be patient because we work in different timezones and asynchronously. Unless the merge request is urgent (like fixing a broken default branch), please don't DM or reassign the merge request before waiting for a 24-hour window. - If you have a question regarding your merge request/issue, make it on the merge request/issue. When we DM each other, we no longer have a SSOT and [no one else is able to contribute](https://about.gitlab.com/handbook/values/#public-by-default). -- When you have a big **Draft** merge request with many changes, you're advised to get the review started before adding/removing significant code. Make sure it is assigned well before the release cut-off, as the reviewer(s)/maintainer(s) would always prioritize reviewing finished MRs before the **Draft** ones. +- When you have a big **Draft** merge request with many changes, you're advised to get the review started before adding/removing significant code. Make sure it is assigned well before the release cut-off, as the reviewers/maintainers would always prioritize reviewing finished MRs before the **Draft** ones. - Make sure to remove the `Draft:` title before the last round of review. ### Share your work early diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index ddd99f3614d..5cfdaff0448 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -400,7 +400,7 @@ We are still learning the best practices for both **type policies** and **reacti Take a moment to improve this guide or [leave a comment](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/100) if you use it! -In the example below we define a `@client` query and its `typedefs`: +In the example below we define a `@client` query and its `typedefs`: ```javascript // ./graphql/typedefs.graphql @@ -1987,7 +1987,7 @@ To improve performance, sometimes we want to make initial GraphQL queries early. } ``` -- Add startup call(s) with correct variables to the HAML file that serves as a view +- Add startup calls with correct variables to the HAML file that serves as a view for your application. To add GraphQL startup calls, we use `add_page_startup_graphql_call` helper where the first parameter is a path to the query, the second one is an object containing query variables. Path to the query is diff --git a/doc/development/fe_guide/registry_architecture.md b/doc/development/fe_guide/registry_architecture.md index 47a6dc40e19..56d67e094b7 100644 --- a/doc/development/fe_guide/registry_architecture.md +++ b/doc/development/fe_guide/registry_architecture.md @@ -56,7 +56,7 @@ in the container components when needed. This makes it easier to: [`delete_package.vue`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/packages_and_registries/package_registry/components/functional/delete_package.vue)). - Leverage [startup for GraphQL calls](graphql.md#making-initial-queries-early-with-graphql-startup-calls). -## Shared compoenents library +## Shared components library Inside `vue_shared/components/registry` and `packages_and_registries/shared`, there's a set of shared components that you can use to implement registry functionality. These components build the diff --git a/doc/development/fe_guide/style/html.md b/doc/development/fe_guide/style/html.md index 72492d56ee4..90ff88bc975 100644 --- a/doc/development/fe_guide/style/html.md +++ b/doc/development/fe_guide/style/html.md @@ -58,11 +58,9 @@ Button tags requires a `type` attribute according to the [W3C HTML specification ### Blank target -Avoid forcing links to open in a new window as this reduces the control the user has over the link. -However, it might be a good idea to use a blank target when replacing the current page with -the link makes the user lose content or progress. +Arbitrarily opening links in a new tab is not recommended, so refer to the [Pajamas guidelines on links](https://design.gitlab.com/product-foundations/interaction/#links) when considering adding `target="_blank"` to links. -Use `rel="noopener noreferrer"` whenever your links open in a new window, that is, `target="_blank"`. This prevents a security vulnerability [documented by JitBit](https://www.jitbit.com/alexblog/256-targetblank---the-most-underestimated-vulnerability-ever/). +When using `target="_blank"` with `a` tags, you must also add the `rel="noopener noreferrer"` attribute. This prevents a security vulnerability [documented by JitBit](https://www.jitbit.com/alexblog/256-targetblank---the-most-underestimated-vulnerability-ever/). When using `gl-link`, using `target="_blank"` is sufficient as it automatically adds `rel="noopener noreferrer"` to the link. diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 1ab97d8a1f5..1c32647eefd 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -155,6 +155,13 @@ $ grep "eslint-disable.*import/no-deprecated" -r . ./app/assets/javascripts/issuable_form.js: // eslint-disable-next-line import/no-deprecated ``` +### GraphQL schema and operations validation + +We use [`@graphql-eslint/eslint-plugin`](https://www.npmjs.com/package/@graphql-eslint/eslint-plugin) +to lint GraphQL schema and operations. This plugin requires the entire schema to function properly. +It is thus recommended to generate an up-to-date dump of the schema when running ESLint locally. +You can do this by running the `./scripts/dump_graphql_schema` script. + ## Formatting with Prettier > Support for `.graphql` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227280) in GitLab 13.2. diff --git a/doc/development/fe_guide/vue3_migration.md b/doc/development/fe_guide/vue3_migration.md index 8c8bb36d962..068b0c5b475 100644 --- a/doc/development/fe_guide/vue3_migration.md +++ b/doc/development/fe_guide/vue3_migration.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w The migration from Vue 2 to 3 is tracked in epic [&6252](https://gitlab.com/groups/gitlab-org/-/epics/6252). -To ease migration to Vue 3.x, we have added [eslint rules](https://gitlab.com/gitlab-org/frontend/eslint-plugin/-/merge_requests/50) +To ease migration to Vue 3.x, we have added [ESLint rules](https://gitlab.com/gitlab-org/frontend/eslint-plugin/-/merge_requests/50) that prevent us from using the following deprecated features in the codebase. ## Vue filters diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index f8f03773c12..68c14c1b0c9 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -262,7 +262,7 @@ To disable a feature flag that has been globally enabled you can run: To disable a feature flag that has been enabled for a specific project you can run: ```shell -/chatops run feature set --group=gitlab-org some_feature false +/chatops run feature set --project=gitlab-org/gitlab some_feature false ``` You cannot selectively disable feature flags for a specific project/group/user without applying a [specific method of implementing](index.md#selectively-disable-by-actor) the feature flags. diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 4b417b26381..54158de6893 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -7,7 +7,7 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo # Feature flags in the development of GitLab -**NOTE**: +NOTE: The documentation below covers feature flags used by GitLab to deploy its own features, which **is not** the same as the [feature flags offered as part of the product](../../operations/feature_flags.md). @@ -154,7 +154,6 @@ This process is meant to ensure consistent feature flag usage in the codebase. A - Be known. Only use feature flags that are explicitly defined. - Not be defined twice. They have to be defined either in FOSS or EE, but not both. - Use a valid and consistent `type:` across all invocations. -- Use the same `default_enabled:` across all invocations. - Have an owner. All feature flags known to GitLab are self-documented in YAML files stored in: @@ -168,7 +167,7 @@ Each feature flag is defined in a separate YAML file consisting of a number of f |---------------------|----------|----------------------------------------------------------------| | `name` | yes | Name of the feature flag. | | `type` | yes | Type of feature flag. | -| `default_enabled` | yes | The default state of the feature flag that is strictly validated, with `default_enabled:` passed as an argument. | +| `default_enabled` | yes | The default state of the feature flag. | | `introduced_by_url` | no | The URL to the merge request that introduced the feature flag. | | `rollout_issue_url` | no | The URL to the Issue covering the feature flag rollout. | | `milestone` | no | Milestone in which the feature was added. | @@ -256,42 +255,10 @@ if Feature.disabled?(:my_feature_flag, project) end ``` -In rare cases you may want to make a feature enabled by default. If so, explain the reasoning -in the merge request. Use `default_enabled: true` when checking the feature flag state: - -```ruby -if Feature.enabled?(:feature_flag, project, default_enabled: true) - # execute code if feature flag is enabled -else - # execute code if feature flag is disabled -end - -if Feature.disabled?(:my_feature_flag, project, default_enabled: true) - # execute code if feature flag is disabled -end -``` - -If not specified, `default_enabled` is `false`. - -To force reading the `default_enabled` value from the relative YAML definition file, use -`default_enabled: :yaml`: - -```ruby -if Feature.enabled?(:feature_flag, project, default_enabled: :yaml) - # execute code if feature flag is enabled -end -``` +Default behavior for not configured feature flags is controlled +by `default_enabled:` in YAML definition. -```ruby -if Feature.disabled?(:feature_flag, project, default_enabled: :yaml) - # execute code if feature flag is disabled -end -``` - -This allows to use the same feature flag check across various parts of the codebase and -maintain the status of `default_enabled` in the YAML definition file which is the SSOT. - -If `default_enabled: :yaml` is used, a YAML definition is expected or an error is raised +If feature flag does not have a YAML definition an error will be raised in development or test environment, while returning `false` on production. If not specified, the default feature flag type for `Feature.enabled?` and `Feature.disabled?` @@ -333,6 +300,17 @@ class MyClass end ``` +#### Recursion detection + +When there are many feature flags, it is not always obvious where they are +called. Avoid cycles where the evaluation of one feature flag requires the +evaluation of other feature flags. If this causes a cycle, it will be broken +and the default value will be returned. + +To enable this recursion detection to work correctly, always access feature values through +`Feature::enabled?`, and avoid the low-level use of `Feature::get`. When this +happens, we track a `Feature::RecursionError` exception to the error tracker. + ### Frontend When using a feature flag for UI elements, make sure to _also_ use a feature @@ -370,16 +348,6 @@ so checking for `gon.features.vim_bindings` would not work. See the [Vue guide](../fe_guide/vue.md#accessing-feature-flags) for details about how to access feature flags in a Vue component. -In rare cases you may want to make a feature enabled by default. If so, explain the reasoning -in the merge request. Use `default_enabled: true` when checking the feature flag state: - -```ruby -before_action do - # Prefer to scope it per project or user e.g. - push_frontend_feature_flag(:vim_bindings, project, default_enabled: true) -end -``` - If not specified, the default feature flag type for `push_frontend_feature_flag` is `type: development`. For all other feature flag types, you must specify the `type:`: @@ -440,6 +408,22 @@ Feature.enabled?(:a_feature, project) && Feature.disabled?(:a_feature_override, /chatops run feature set --project=gitlab-org/gitlab a_feature_override true ``` +#### Use actors for verifying in production + +WARNING: +Using production as a testing environment is not recommended. Use our testing +environments for testing features that are not production-ready. + +While the staging environment provides a way to test features in an environment +that resembles production, it doesn't allow you to compare before-and-after +performance metrics specific to production environment. It can be useful to have a +project in production with your development feature flag enabled, to allow tools +like Sitespeed reports to reveal the metrics of the new code under a feature flag. + +This approach is even more useful if you're already tracking the old codebase in +Sitespeed, enabling you to compare performance accurately before and after the +feature flag's rollout. + ### Enable additional objects as actors To use feature gates based on actors, the model needs to respond to @@ -673,9 +657,7 @@ You can control whether the `Flipper::Adapters::Memory` or `ActiveRecord` mode i #### `stub_feature_flags: true` (default and preferred) In this mode Flipper is configured to use `Flipper::Adapters::Memory` and mark all feature -flags to be on-by-default and persisted on a first use. This overwrites the `default_enabled:` -of `Feature.enabled?` and `Feature.disabled?` returning always `true` unless feature flag -is persisted. +flags to be on-by-default and persisted on a first use. Make sure behavior under feature flag doesn't go untested in some non-specific contexts. diff --git a/doc/development/features_inside_dot_gitlab.md b/doc/development/features_inside_dot_gitlab.md index 7b11b541b5a..ca7dbd6adde 100644 --- a/doc/development/features_inside_dot_gitlab.md +++ b/doc/development/features_inside_dot_gitlab.md @@ -16,7 +16,6 @@ When implementing new features, please refer to these existing features to avoid - [CODEOWNERS](../user/project/code_owners.md#set-up-code-owners): `.gitlab/CODEOWNERS`. - [Route Maps](../ci/review_apps/#route-maps): `.gitlab/route-map.yml`. - [Customize Auto DevOps Helm Values](../topics/autodevops/customize.md#customize-values-for-helm-chart): `.gitlab/auto-deploy-values.yaml`. -- [GitLab managed apps CI/CD](../user/clusters/applications.md#prerequisites): `.gitlab/managed-apps/config.yaml`. - [Insights](../user/project/insights/index.md#configure-your-insights): `.gitlab/insights.yml`. - [Service Desk Templates](../user/project/service_desk.md#using-customized-email-templates): `.gitlab/service_desk_templates/`. - [Web IDE](../user/project/web_ide/#web-ide-configuration-file): `.gitlab/.gitlab-webide.yml`. diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index 8fe5af56f9d..d4274c6275b 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -97,3 +97,414 @@ virtual machine: ```shell fips-mode-setup --disable ``` + +#### Detect FIPS enablement in code + +You can query `GitLab::FIPS` in Ruby code to determine if the instance is FIPS-enabled: + +```ruby +def default_min_key_size(name) + if Gitlab::FIPS.enabled? + Gitlab::SSHPublicKey.supported_sizes(name).select(&:positive?).min || -1 + else + 0 + end +end +``` + +## Nightly Omnibus FIPS builds + +The Distribution team has created [nightly FIPS Omnibus builds](https://packages.gitlab.com/gitlab/nightly-fips-builds). These +GitLab builds are compiled to use the system OpenSSL instead of the Omnibus-embedded version of OpenSSL. + +See [the section on how FIPS builds are created](#how-fips-builds-are-created). + +## Runner + +See the [documentation on installing a FIPS-compliant GitLab Runner](https://docs.gitlab.com/runner/install/#fips-compliant-gitlab-runner). + +## Set up a FIPS-enabled cluster + +You can use the [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) to spin +up a FIPS-enabled cluster for development and testing. These instructions use Amazon Web Services (AWS) +because that is the first target environment, but you can adapt them for other providers. + +### Set up your environment + +To get started, your AWS account must subscribe to a FIPS-enabled Amazon +Machine Image (AMI) in the [AWS Marketplace console](https://aws.amazon.com/premiumsupport/knowledge-center/launch-ec2-marketplace-subscription/). + +This example assumes that the `Ubuntu Pro 20.04 FIPS LTS` AMI by +`Canonical Group Limited` has been added your account. This operating +system is used for virtual machines running in Amazon EC2. + +### Omnibus + +The simplest way to get a FIPS-enabled GitLab cluster is to use an Omnibus reference architecture. +See the [GET Quick Start Guide](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/docs/environment_quick_start_guide.md) +for more details. The following instructions build on the Quick Start and are also necessary for [Cloud Native Hybrid](#cloud-native-hybrid) installations. + +#### Terraform: Use a FIPS AMI + +1. Follow the guide to set up Terraform and Ansible. +1. After [step 2b](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/docs/environment_quick_start_guide.md#2b-setup-config), + create a `data.tf` in your environment (for example, `gitlab-environment-toolkit/terraform/environments/gitlab-10k/inventory/data.tf`): + + ```tf + data "aws_ami" "ubuntu_20_04_fips" { + count = 1 + + most_recent = true + + filter { + name = "name" + values = ["ubuntu-pro-fips-server/images/hvm-ssd/ubuntu-focal-20.04-amd64-pro-fips-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["aws-marketplace"] + } + ``` + +1. Add the custom `ami_id` to use this AMI in `environment.tf`. For + example, in `gitlab-environment-toolkit/terraform/environments/gitlab-10k/inventory/environment.tf`: + + ```tf + module "gitlab_ref_arch_aws" { + source = "../../modules/gitlab_ref_arch_aws" + + prefix = var.prefix + ami_id = data.aws_ami.ubuntu_20_04_fips[0].id + ... + ``` + +NOTE: +GET does not allow the AMI to change on EC2 instances after it has +been deployed via `terraform apply`. Since an AMI change would tear down +an instance, this would result in data loss: not only would disks be +destroyed, but also GitLab secrets would be lost. There is a [Terraform lifecycle rule](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/blob/2aaeaff8ac8067f23cd7b6bb5bf131061649089d/terraform/modules/gitlab_aws_instance/main.tf#L40) +to ignore AMI changes. + +#### Ansible: Specify the FIPS Omnibus builds + +The standard Omnibus GitLab releases build their own OpenSSL library, +which is not FIPS-validated. However, we have nightly builds that create +Omnibus packages that link against the operating system's OpenSSL library. To +use this package, update the `gitlab_repo_script_url` field in the +Ansible `vars.yml`. For example, you might modify +`gitlab-environment-toolkit/ansible/environments/gitlab-10k/inventory/vars.yml` +in this way: + +```yaml +all: + vars: + ... + gitlab_repo_script_url: "https://packages.gitlab.com/install/repositories/gitlab/nightly-fips-builds/script.deb.sh" +``` + +### Cloud Native Hybrid + +A Cloud Native Hybrid install uses both Omnibus and Cloud Native GitLab +(CNG) images. The previous instructions cover the Omnibus part, but two +additional steps are needed to enable FIPS in CNG: + +1. Use a custom Amazon Elastic Kubernetes Service (EKS) AMI. +1. Use GitLab containers built with RedHat's Universal Base Image (UBI). + +#### Build a custom EKS AMI + +Because Amazon does not yet publish a FIPS-enabled AMI, you have to +build one yourself with Packer. + +Amazon publishes the following Git repositories with information about custom EKS AMIs: + +- [Amazon EKS AMI Build Specification](https://github.com/awslabs/amazon-eks-ami) +- [Sample EKS custom AMIs](https://github.com/aws-samples/amazon-eks-custom-amis/) + +This [GitHub pull request](https://github.com/awslabs/amazon-eks-ami/pull/898) makes +it possible to create an Amazon Linux 2 EKS AMI with FIPS enabled for Kubernetes v1.21. +To build an image: + +1. [Install Packer](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli). +1. Run the following: + + ```shell + git clone https://github.com/awslabs/amazon-eks-ami + cd amazon-eks-ami + git fetch origin pull/898/head:fips-ami + git checkout fips-ami + AWS_DEFAULT_REGION=us-east-1 make 1.21-fips # Be sure to set the region accordingly + ``` + +If you are using a different version of Kubernetes, adjust the `make` +command and `Makefile` accordingly. + +When the AMI build is done, a new AMI should be created with a message +such as the following: + +```plaintext +==> Builds finished. The artifacts of successful builds are: +--> amazon-ebs: AMIs were created: +us-west-2: ami-0a25e760cd00b027e +``` + +In this example, the AMI ID is `ami-0a25e760cd00b027e`, but your value may +be different. + +Building a RHEL-based system with FIPS enabled should be possible, but +there is [an outstanding issue preventing the Packer build from completing](https://github.com/aws-samples/amazon-eks-custom-amis/issues/51). + +#### Terraform: Use a custom EKS AMI + +Now you can set the custom EKS AMI. + +1. In `environment.tf`, add `eks_ami_id = var.eks_ami_id` so you can pass this variable to the + AWS reference architecture module. For example, in + `gitlab-environment-toolkit/terraform/environments/gitlab-10k/inventory/environment.tf`: + + ```tf + module "gitlab_ref_arch_aws" { + source = "../../modules/gitlab_ref_arch_aws" + + prefix = var.prefix + ami_id = data.aws_ami.ubuntu_20_04_fips[0].id + eks_ami_id = var.eks_ami_id + .... + ``` + +1. In `variables.tf`, define a `eks_ami_id` with the AMI ID in the + previous step: + + ```tf + variable "eks_ami_id" { + default = "ami-0a25e760cd00b027e" + } + ``` + +#### Ansible: Use UBI images + +CNG uses a Helm Chart to manage which container images to deploy. By default, GET +deploys the latest released versions that use Debian-based containers. + +To switch to UBI-based containers, edit the Ansible `vars.yml` to use custom +Charts variables: + +```yaml +all: + vars: + ... + gitlab_charts_custom_config_file: '/path/to/gitlab-environment-toolkit/ansible/environments/gitlab-10k/inventory/charts.yml' +``` + +Now create `charts.yml` in the location specified above and specify tags with a `-ubi8` suffix. For example: + +```yaml +global: + image: + pullPolicy: Always + certificates: + image: + tag: master-ubi8 + +gitlab: + gitaly: + image: + tag: master-ubi8 + gitlab-exporter: + image: + tag: master-ubi8 + gitlab-shell: + image: + tag: main-ubi8 # The default branch is main, not master + gitlab-mailroom: + image: + tag: master-ubi8 + migrations: + image: + tag: master-ubi8 + sidekiq: + image: + tag: master-ubi8 + toolbox: + image: + tag: master-ubi8 + webservice: + image: + tag: master-ubi8 + workhorse: + tag: master-ubi8 + +nginx-ingress: + controller: + image: + repository: registry.gitlab.com/stanhu/gitlab-test-images/k8s-staging-ingress-nginx/controller + tag: v1.2.0-beta.1 + pullPolicy: Always + digest: sha256:ace38833689ad34db4a46bc1e099242696eb800def88f02200a8615530734116 +``` + +The above example shows a FIPS-enabled [`nginx-ingress`](https://github.com/kubernetes/ingress-nginx) image. +See [this issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3153#note_917782207) for more details on +how to build NGINX and the Ingress Controller. + +You can also use release tags, but the versioning is tricky because each +component may use its own versioning scheme. For example, for GitLab v14.10: + +```yaml +global: + certificates: + image: + tag: 20191127-r2-ubi8 + +gitlab: + gitaly: + image: + tag: v14.10.0-ubi8 + gitlab-exporter: + image: + tag: 11.14.0-ubi8 + gitlab-shell: + image: + tag: v13.25.1-ubi8 + gitlab-mailroom: + image: + tag: v14.10.0-ubi8 + migrations: + image: + tag: v14.10.0-ubi8 + sidekiq: + image: + tag: v14.10.0-ubi8 + toolbox: + image: + tag: v14.10.0-ubi8 + webservice: + image: + tag: v14.10.0-ubi8 + workhorse: + tag: v14.10.0-ubi8 +``` + +## Verify FIPS + +The following sections describe ways you can verify if FIPS is enabled. + +### Kernel + +```shell +$ cat /proc/sys/crypto/fips_enabled +1 +``` + +### Ruby (Omnibus images) + +```ruby +$ /opt/gitlab/embedded/bin/irb +irb(main):001:0> require 'openssl'; OpenSSL.fips_mode +=> true +``` + +### Ruby (CNG images) + +```ruby +$ irb +irb(main):001:0> require 'openssl'; OpenSSL.fips_mode +=> true +``` + +### Go + +Google maintains a [`dev.boringcrypto` branch](https://github.com/golang/go/tree/dev.boringcrypto) in the Golang compiler +that makes it possible to statically link BoringSSL, a FIPS-validated module forked from OpenSSL. +However, BoringSSL is not intended for public use. + +We use [`golang-fips`](https://github.com/golang-fips/go), [a fork of the `dev.boringcrypto` branch](https://github.com/golang/go/blob/2fb6bf8a4a51f92f98c2ae127eff2b7ac392c08f/README.boringcrypto.md) to build Go programs that +[dynamically link OpenSSL via `dlopen`](https://github.com/golang-fips/go/blob/go1.18.1-1-openssl-fips/src/crypto/internal/boring/boring.go#L47-L65). This has several advantages: + +- Using a FIPS-validated, system OpenSSL is straightforward. +- This is the source code used by [Red Hat's go-toolset package](https://gitlab.com/redhat/centos-stream/rpms/golang#sources). +- Unlike [go-toolset](https://developers.redhat.com/blog/2019/06/24/go-and-fips-140-2-on-red-hat-enterprise-linux#), this fork appears to keep up with the latest Go releases. + +However, [cgo](https://pkg.go.dev/cmd/cgo) must be enabled via `CGO_ENABLED=1` for this to work. There +is a performance hit when calling into C code. + +Projects that are compiled with `golang-fips` on Linux x86 automatically +get built the crypto routines that use OpenSSL. While the `boringcrypto` +build tag is automatically present, no extra build tags are actually +needed. There are [specific build tags](https://github.com/golang-fips/go/blob/go1.18.1-1-openssl-fips/src/crypto/internal/boring/boring.go#L6) +that disable these crypto hooks. + +We can [check whether a given binary is using OpenSSL](https://go.googlesource.com/go/+/dev.boringcrypto/misc/boring/#caveat) via `go tool nm` +and look for symbols named `Cfunc__goboringcrypto`. For example: + +```plaintext +$ go tool nm nginx-ingress-controller | grep Cfunc__goboringcrypto | tail + 2a0b650 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA384_Final + 2a0b658 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA384_Init + 2a0b660 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA384_Update + 2a0b668 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA512_Final + 2a0b670 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA512_Init + 2a0b678 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_SHA512_Update + 2a0b680 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_internal_ECDSA_sign + 2a0b688 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_internal_ECDSA_verify + 2a0b690 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_internal_ERR_error_string_n + 2a0b698 D crypto/internal/boring._cgo_71ae3cd1ca33_Cfunc__goboringcrypto_internal_ERR_get_error +``` + +In addition, LabKit contains routines to [check whether FIPS is enabled](https://gitlab.com/gitlab-org/labkit/-/tree/master/fips). + +## How FIPS builds are created + +Many GitLab projects (for example: Gitaly, GitLab Pages) have +standardized on using `FIPS_MODE=1 make` to build FIPS binaries locally. + +### Omnibus + +The Omnibus FIPS builds are triggered with the `USE_SYSTEM_SSL` +environment variable set to `true`. When this environment variable is +set, the Omnibus recipes dependencies such as `curl`, NGINX, and libgit2 +will link against the system OpenSSL. OpenSSL will NOT be included in +the Omnibus build. + +The Omnibus builds are created using container images [that use the `golang-fips` compiler](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/blob/master/docker/snippets/go_fips). For +example, [this job](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/jobs/2363742108) created +the `registry.gitlab.com/gitlab-org/gitlab-omnibus-builder/centos_8_fips:3.3.1` image used to +build packages for RHEL 8. + +#### Add a new FIPS build for another Linux distribution + +First, you need to make sure there is an Omnibus builder image for the +desired Linux distribution. The images used to build Omnibus packages are +created with [Omnibus Builder images](https://gitlab.com/gitlab-org/gitlab-omnibus-builder). + +Review [this merge request](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/218). A +new image can be added by: + +1. Adding CI jobs with the `_fips` suffix (for example: `ubuntu_18.04_fips`). +1. Making sure the `Dockerfile` uses `Snippets.new(fips: fips).populate` instead of `Snippets.new.populate`. + +After this image has been tagged, add a new [CI job to Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/911fbaccc08398dfc4779be003ea18014b3e30e9/gitlab-ci-config/dev-gitlab-org.yml#L594-602). + +### Cloud Native GitLab (CNG) + +The Cloud Native GitLab CI pipeline generates images using several base images: + +- Debian +- [Red Hat's Universal Base Image (UBI)](https://developers.redhat.com/products/rhel/ubi) + +UBI images ship with the same OpenSSL package as those used by +RHEL. This makes it possible to build FIPS-compliant binaries without +needing RHEL. Note that RHEL 8.2 ships a [FIPS-validated OpenSSL](https://access.redhat.com/articles/2918071), but 8.5 is in +review for FIPS validation. + +[This merge request](https://gitlab.com/gitlab-org/build/CNG/-/merge_requests/981) +introduces a FIPS pipeline for CNG images. Images tagged for FIPS have the `-fips` suffix. For example, +the `webservice` container has the following tags: + +- `master` +- `master-ubi8` +- `master-fips` diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md index db8367fe5f5..c20c70623ae 100644 --- a/doc/development/foreign_keys.md +++ b/doc/development/foreign_keys.md @@ -123,3 +123,7 @@ class UserConfig < ActiveRecord::Base belongs_to :user end ``` + +Using a foreign key as primary key saves space but can make +[batch counting](service_ping/implement.md#batch-counters) in [Service Ping](service_ping/index.md) less efficient. +Consider using a regular `id` column if the table will be relevant for Service Ping. diff --git a/doc/development/geo.md b/doc/development/geo.md index f37901754aa..f62b2de30db 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -97,7 +97,7 @@ projects that need updating. Those projects can be: timestamp that is more recent than the `last_repository_successful_sync_at` timestamp in the `Geo::ProjectRegistry` model. - Manual: The administrator can manually flag a repository to resync in the - [Geo admin panel](../user/admin_area/geo_nodes.md). + [Geo Admin Area](../user/admin_area/geo_nodes.md). When we fail to fetch a repository on the secondary `RETRIES_BEFORE_REDOWNLOAD` times, Geo does a so-called _re-download_. It will do a clean clone @@ -118,17 +118,6 @@ CI Job Artifacts and LFS objects are synced in a similar way as uploads, but they are tracked by `Geo::JobArtifactRegistry`, and `Geo::LfsObjectRegistry` models respectively. -#### File Download Dispatch worker - -Also similar to the [Repository Sync worker](#repository-sync-worker), -there is a `Geo::FileDownloadDispatchWorker` class that is run -periodically to sync all uploads that aren't synced to the Geo -**secondary** site yet. - -Files are copied via HTTP(s) and initiated via the -`/api/v4/geo/transfers/:type/:id` endpoint, -for example, `/api/v4/geo/transfers/lfs/123`. - ## Authentication To authenticate file transfers, each `GeoNode` record has two fields: @@ -212,7 +201,7 @@ rails g geo_migration [args] [options] To migrate the tracking database, run: ```shell -bundle exec rake geo:db:migrate +bundle exec rake db:migrate:geo ``` ## Finders @@ -259,7 +248,7 @@ basically hashes all Git refs together and stores that hash in the The **secondary** site does the same to calculate the hash of its clone, and compares the hash with the value the **primary** site calculated. If there is a mismatch, Geo will mark this as a mismatch -and the administrator can see this in the [Geo admin panel](../user/admin_area/geo_nodes.md). +and the administrator can see this in the [Geo Admin Area](../user/admin_area/geo_nodes.md). ## Glossary diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index 778387986d8..055c2cd4ea8 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -93,12 +93,6 @@ module Geo def self.model ::Packages::PackageFile end - - # The feature flag follows the format `geo_#{replicable_name}_replication`, - # so here it would be `geo_package_file_replication` - def self.replication_enabled_by_default? - false - end end end ``` diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 275e9421983..0743a03ddac 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -63,8 +63,7 @@ in This should make it easier to contribute for developers who are less comfortable writing Go code. -There is documentation for this approach in [the Gitaly -repository](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/ruby_endpoint.md). +For more information, see the [Beginner's guide to Gitaly contributions](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/beginners_guide.md). ## Gitaly-Related Test Failures @@ -372,3 +371,20 @@ the integration by using GDK: ```shell curl --silent "http://localhost:9236/metrics" | grep go_find_all_tags ``` + +## Using Praefect in test + +By default Praefect in test uses an in-memory election strategy. This strategy +is deprecated and no longer used in production. It mainly is kept for +unit-testing purposes. + +A more modern election strategy requires a connection with a PostgreSQL +database. This behavior is disabled by default when running tests, but you can +enable it by setting `GITALY_PRAEFECT_WITH_DB=1` in your environment. + +This requires you have PostgreSQL running, and you have the database created. +When you are using GDK, you can set it up with: + +1. Start the database: `gdk start db` +1. Load the environment from GDK: `eval $(cd ../gitaly && gdk env)` +1. Create the database: `createdb --encoding=UTF8 --locale=C --echo praefect_test` diff --git a/doc/development/gitlab_flavored_markdown/index.md b/doc/development/gitlab_flavored_markdown/index.md index 682d8011cd8..7f7781cbc62 100644 --- a/doc/development/gitlab_flavored_markdown/index.md +++ b/doc/development/gitlab_flavored_markdown/index.md @@ -4,13 +4,13 @@ group: Editor info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Markdown developer documentation **(FREE)** +# GitLab Flavored Markdown (GLFM) developer documentation **(FREE)** -This page contains the MVC for the developer documentation for GitLab Flavored Markdown. +This page contains the MVC for the developer documentation for GitLab Flavored Markdown (GLFM). For the user documentation about Markdown in GitLab, refer to [GitLab Flavored Markdown](../../user/markdown.md). -## GitLab Flavored Markdown specification guide +## GitLab Flavored Markdown (GLFM) specification guide The [specification guide](specification_guide/index.md) includes: @@ -18,3 +18,4 @@ The [specification guide](specification_guide/index.md) includes: - [Parsing and rendering](specification_guide/index.md#parsing-and-rendering). - [Goals](specification_guide/index.md#goals). - [Implementation](specification_guide/index.md#implementation) of the spec. +- [Workflows](specification_guide/index.md#workflows). diff --git a/doc/development/gitlab_flavored_markdown/specification_guide/index.md b/doc/development/gitlab_flavored_markdown/specification_guide/index.md index 021f7bafce9..397d555c54f 100644 --- a/doc/development/gitlab_flavored_markdown/specification_guide/index.md +++ b/doc/development/gitlab_flavored_markdown/specification_guide/index.md @@ -64,7 +64,7 @@ serve as input to automated conformance tests. It is [explained in the CommonMark specification](https://spec.commonmark.org/0.30/#about-this-document): > This document attempts to specify Markdown syntax unambiguously. It contains many -> examples with side-by-side Markdown and HTML. These are intended to double as conformance tests. +> examples with side-by-side Markdown and HTML. These examples are intended to double as conformance tests. The HTML-rendered versions of the specifications: @@ -385,20 +385,61 @@ subgraph output:<br/>GLFM specification files end ``` +#### `canonicalize-html.rb` script + +The `scripts/glfm/canonicalize-html.rb` handles the +["canonicalization" of HTML](#canonicalization-of-html). It is a pipe-through +helper script which takes as input a static or WYSIWYG HTML string containing +extra HTML, and outputs a canonical HTML string. + +It is implemented as a standalone, modular, single-purpose script, based on the +[Unix philosophy](https://en.wikipedia.org/wiki/Unix_philosophy#:~:text=The%20Unix%20philosophy%20emphasizes%20building,developers%20other%20than%20its%20creators.). +It's easy to use when running the standard CommonMark `spec_tests.py` +script, which expects canonical HTML, against the GitLab renderer implementations. + +#### `run-spec-tests.sh` script + +`scripts/glfm/run-spec-tests.sh` is a convenience shell script which runs +conformance specs via the CommonMark standard `spec_tests.py` script, +which uses the `glfm_specification/output/spec.txt` file and `scripts/glfm/canonicalize-html.rb` +helper script to test the GLFM renderer implementations' support for rendering Markdown +specification examples to canonical HTML. + +```mermaid +graph LR +subgraph scripts: + A{run-spec-tests.sh} --> C + subgraph specification testing process + B[canonicalize-html.sh] --> C + C[spec_tests.py] + end +end +subgraph input + D[spec.txt GLFM specification] --> C + E((GLFM static<br/>renderer implementation)) --> B + F((GLFM WYSIWYG<br/>renderer implementation)) --> B +end +subgraph output:<br/>test results/output + C --> G[spec_tests.py output] +end +``` + #### `update-example-snapshots.rb` script -The `scripts/glfm/update-example-snapshots.rb` script uses input specification -files to update example snapshots: +The `scripts/glfm/update-example-snapshots.rb` script uses the GLFM +`glfm_specification/output/spec.txt` specification file and the +`glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml` +file to create and update the [example snapshot](#example-snapshot-files) +YAML files: ```mermaid graph LR subgraph script: A{update-example-snapshots.rb} end -subgraph input:<br/>input specification files - B[downloaded gfm_spec_v_0.29.txt] --> A - C[glfm_canonical_examples.txt] --> A - D[glfm_example_status.yml] --> A +subgraph input:<br/>input specification file + B[spec.txt] --> A + C[glfm_example_status.yml] --> A end subgraph output:<br/>example snapshot files A --> E[examples_index.yml] @@ -435,9 +476,11 @@ code. It contains only shell scripting commands for the relevant ```mermaid graph LR +subgraph tests: + B[relevant rspec+jest test files] +end subgraph script: - A{run-snapshopt-tests.sh} --> B - B[relevant rspec/jest test files] + A{run-snapshopt-tests.sh} -->|invokes| B end subgraph input:<br/>YAML C[examples_index.yml] --> B @@ -446,46 +489,7 @@ subgraph input:<br/>YAML F[prosemirror_json.yml] --> B end subgraph output:<br/>test results/output - B --> G[rspec/jest output] -end -``` - -#### `canonicalize-html.rb` script - -The `scripts/glfm/canonicalize-html.rb` handles the -["canonicalization" of HTML](#canonicalization-of-html). It is a pipe-through -helper script which takes as input a static or WYSIWYG HTML string containing -extra HTML, and outputs a canonical HTML string. - -It is implemented as a standalone, modular, single-purpose script, based on the -[Unix philosophy](https://en.wikipedia.org/wiki/Unix_philosophy#:~:text=The%20Unix%20philosophy%20emphasizes%20building,developers%20other%20than%20its%20creators.). -It's easy to use when running the standard CommonMark `spec_tests.py` -script, which expects canonical HTML, against the GitLab renderer implementations. - -#### `run-spec-tests.sh` script - -`scripts/glfm/run-spec-tests.sh` is a convenience shell script which runs -conformance specs via the CommonMark standard `spec_tests.py` script, -which uses the `glfm_specification/output/spec.txt` file and `scripts/glfm/canonicalize-html.rb` -helper script to test the GLFM renderer implementations' support for rendering Markdown -specification examples to canonical HTML. - -```mermaid -graph LR -subgraph scripts: - A{run-spec-tests.sh} --> C - subgraph specification testing process - B[canonicalize-html.sh] --> C - C[spec_tests.py] - end -end -subgraph input - D[spec.txt GLFM specification] --> C - E((GLFM static<br/>renderer implementation)) --> B - F((GLFM WYSIWYG<br/>renderer implementation)) --> B -end -subgraph output:<br/>test results/output - C --> G[spec_tests.py output] + B --> H[rspec+jest output] end ``` @@ -506,21 +510,76 @@ They are either downloaded, as in the case of the GFM `spec.txt` file, or manually updated, as in the case of all GFM files. -- `glfm_specification/input/github_flavored_markdown/gfm_spec_v_0.29.txt` - - official latest [GFM spec.txt](https://github.com/github/cmark-gfm/blob/master/test/spec.txt), - automatically downloaded and updated by `update-specification.rb` script. -- `glfm_specification/input/gitlab_flavored_markdown/glfm_intro.txt` - - Manually updated text of intro section for generated GLFM `spec.txt`. - - Replaces GFM version of introductory - section in `spec.txt`. -- `glfm_specification/input/gitlab_flavored_markdown/glfm_canonical_examples.txt` - - Manually updated canonical Markdown+HTML examples for GLFM extensions. - - Standard backtick-delimited `spec.txt` examples format with Markdown + canonical HTML. - - Inserted as a new section before the appendix of generated `spec.txt`. -- `glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml` - - Manually updated status of automatic generation of files based on Markdown - examples. - - Allows example snapshot generation, Markdown conformance tests, or +##### GitHub Flavored Markdown specification + +[`glfm_specification/input/github_flavored_markdown/gfm_spec_v_0.29.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/github_flavored_markdown/gfm_spec_v_0.29.txt) +is the official latest [GFM `spec.txt`](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). + +- It is automatically downloaded and updated by `update-specification.rb` script. +- When it is downloaded, the version number is added to the filename. + +##### `glfm_intro.txt` + +[`glfm_specification/input/gitlab_flavored_markdown/glfm_intro.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/gitlab_flavored_markdown/glfm_intro.txt) +is the GitLab-specific version of the prose in the introduction section of the GLFM specification. + +- It is manually updated. +- The `update-specification.rb` script inserts it into the generated GLFM `spec.txt` to replace + the GitHub-specific GFM version of the introductory section. + +##### `glfm_canonical_examples.txt` + +[`glfm_specification/input/gitlab_flavored_markdown/glfm_canonical_examples.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/gitlab_flavored_markdown/glfm_canonical_examples.txt) +is the manually updated canonical Markdown+HTML examples for GLFM extensions. + +- It contains examples in the [standard backtick-delimited `spec.txt` format](#various-markdown-specifications), + each of which contain a Markdown example and the corresponding canonical HTML. +- The `update-specification.rb` script inserts it as new sections before the appendix + of generated `spec.txt`. +- It should consist of `H1` header sections, with all examples nested exactly 2 levels deep within `H2` + header sections. + +`glfm_specification/input/gitlab_flavored_markdown/glfm_canonical_examples.txt` sample entries: + +NOTE: +All lines in this example are prefixed with a `|` character. This prefix helps avoid false +errors when this file is checked by `markdownlint`, and possible errors in other Markdown editors. +The actual file should not have these prefixed `|` characters. + +```plaintext +|# First GitLab-Specific Section with Examples +| +|## Strong but with two asterisks +| +|```````````````````````````````` example +|**bold** +|. +|<p><strong>bold</strong></p> +|```````````````````````````````` +| +|# Second GitLab-Specific Section with Examples +| +|## Strong but with HTML +| +|```````````````````````````````` example +|<strong> +|bold +|</strong> +|. +|<p><strong> +|bold +|</strong></p> +|```````````````````````````````` +``` + +##### `glfm_example_status.yml` + +[`glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml) +controls the behavior of the [scripts](#scripts) and [tests](#types-of-markdown-tests-driven-by-the-glfm-specification). + +- It is manually updated. +- It controls the status of automatic generation of files based on Markdown examples. +- It allows example snapshot generation, Markdown conformance tests, or Markdown snapshot tests to be skipped for individual examples. For example, if they are unimplemented, broken, or cannot be tested for some reason. @@ -528,12 +587,14 @@ updated, as in the case of all GFM files. ```yaml 07_99_an_example_with_incomplete_wysiwyg_implementation_1: - skip_update_example_snapshots: true - skip_running_snapshot_static_html_tests: false - skip_running_snapshot_wysiwyg_html_tests: true - skip_running_snapshot_prosemirror_json_tests: true + skip_update_example_snapshots: false + skip_update_example_snapshot_html_static: false + skip_update_example_snapshot_html_wysiwyg: false skip_running_conformance_static_tests: false - skip_running_conformance_wysiwyg_tests: true + skip_running_conformance_wysiwyg_tests: false + skip_running_snapshot_static_html_tests: false + skip_running_snapshot_wysiwyg_html_tests: false + skip_running_snapshot_prosemirror_json_tests: false ``` #### Output specification files @@ -541,28 +602,39 @@ updated, as in the case of all GFM files. The `glfm_specification/output` directory contains the CommonMark standard format `spec.txt` file which represents the canonical GLFM specification which is generated by the `update-specification.rb` script. It also contains the rendered `spec.html` -and `spec.pdf` which are generated from with the `spec.txt` as input. - -- `glfm_specification/output/spec.txt` - A Markdown file, in the standard format - with prose and Markdown + canonical HTML examples, generated (or updated) by the - `update-specification.rb` script. -- `glfm_specification/output/spec.html` - An HTML file, rendered based on `spec.txt`, - also generated (or updated) by the `update-specification.rb` script at the same time as - `spec.txt`. It corresponds to the HTML-rendered versions of the - "GitHub Flavored Markdown" (<abbr title="GitHub Flavored Markdown">GFM</abbr>) - [specification](https://github.github.com/gfm/) - and the [CommonMark specification](https://spec.commonmark.org/0.30/). - -These output `spec.**` files, which represent the official, canonical GLFM specification +which is generated based on the `spec.txt` as input. + +These output `spec.*` files, which represent the official, canonical GLFM specification, are colocated under the same parent folder `glfm_specification` with the other `input` specification files. They're located here both for convenience, and because they are all -a mix of manually edited and generated files. In GFM, -`spec.txt` is [located in the test dir](https://github.com/github/cmark-gfm/blob/master/test/spec.txt), -and in CommonMark it's located -[in the project root](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). -No precedent exists for a standard location. In the future, we may decide to +a mix of manually edited and generated files. + +In GFM, `spec.txt` is [located in the test dir](https://github.com/github/cmark-gfm/blob/master/test/spec.txt), +and in CommonMark it's located [in the project root](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). No precedent exists for a standard location. In the future, we may decide to move or copy a hosted version of the rendered HTML `spec.html` version to another location or site. +##### spec.txt + +[`glfm_specification/output/spec.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/output/spec.txt) +is a Markdown specification file, in the standard format +with prose and Markdown + canonical HTML examples. It is generated or updated by the +`update-specification.rb` script. + +It also serves as input for other scripts such as `update-example-snapshots.rb` +and `run-spec-tests.sh`. + +##### spec.html + +[`glfm_specification/output/spec.html`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/output/spec.html) +is an HTML file, rendered based on `spec.txt`. It is +also generated (or updated) by the `update-specification.rb` script at the same time as +`spec.txt`. + +It corresponds to the HTML-rendered versions of the +"GitHub Flavored Markdown" (<abbr title="GitHub Flavored Markdown">GFM</abbr>) +[specification](https://github.github.com/gfm/) +and the [CommonMark specification](https://spec.commonmark.org/0.30/). + ### Example snapshot files The `example_snapshots` directory contains files which are generated by the @@ -574,12 +646,13 @@ After the entire GLFM implementation is complete for both backend (Ruby) and frontend (JavaScript), all of these YAML files can be automatically generated. However, while the implementations are still in progress, the `skip_update_example_snapshots` key in `glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml` -can be used to disable automatic generation of some examples, and they can instead +can be used to disable automatic generation of some examples. They can instead be manually edited as necessary to help drive the implementations. #### `spec/fixtures/glfm/example_snapshots/examples_index.yml` -`spec/fixtures/glfm/example_snapshots/examples_index.yml` is the main list of all +[`spec/fixtures/glfm/example_snapshots/examples_index.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/fixtures/glfm/example_snapshots/examples_index.yml) +is the main list of all CommonMark, GFM, and GLFM example names, each with a unique canonical name. - It is generated from the hierarchical sections and examples in the @@ -590,10 +663,15 @@ CommonMark, GFM, and GLFM example names, each with a unique canonical name. the additional Section 7 in the GLFM `spec.txt`. - It also contains extra metadata about each example, such as: 1. `spec_txt_example_position` - The position of the example in the generated GLFM `spec.txt` file. + - This value is the index order of each individual Markdown + HTML5 example in the file. It is _not_ + the line number in the file. + - This value can be used to locate the example in the rendered `spec.html` file, because the standard + CommonMark tooling includes the index number for each example in the rendered HTML file. + For example: [https://spec.commonmark.org/0.30/#example-42](https://spec.commonmark.org/0.30/#example-42) 1. `source_specification` - Which specification the example originally came from: `commonmark`, `github`, or `gitlab`. - The naming convention for example entry names is based on nested header section - names and example index within the header. + names and example index in the header. - This naming convention should result in fairly stable names and example positions. The CommonMark / GLFM specification rarely changes, and most GLFM examples where multiple examples exist for the same Section 7 subsection are @@ -621,7 +699,7 @@ CommonMark, GFM, and GLFM example names, each with a unique canonical name. #### `spec/fixtures/glfm/example_snapshots/markdown.yml` -`spec/fixtures/glfm/example_snapshots/markdown.yml` contains the original Markdown +[`spec/fixtures/glfm/example_snapshots/markdown.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/fixtures/glfm/example_snapshots/markdown.yml) contains the original Markdown for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` - For CommonMark and GFM Markdown, @@ -634,14 +712,14 @@ for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` `spec/fixtures/glfm/example_snapshots/markdown.yml` sample entry: ```yaml -06_04_inlines_emphasis_and_strong_emphasis_1: |- +06_04_inlines_emphasis_and_strong_emphasis_1: | *foo bar* ``` #### `spec/fixtures/glfm/example_snapshots/html.yml` -`spec/fixtures/glfm/example_snapshots/html.yml` contains the HTML for each entry in -`spec/fixtures/glfm/example_snapshots/examples_index.yml` +[`spec/fixtures/glfm/example_snapshots/html.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/fixtures/glfm/example_snapshots/html.yml) +contains the HTML for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` Three types of entries exist, with different HTML for each: @@ -670,11 +748,11 @@ Any exceptions or failures which occur when generating HTML are replaced with an ```yaml 06_04_inlines_emphasis_and_strong_emphasis_1: - canonical: |- + canonical: | <p><em>foo bar</em></p> - static: |- + static: | <p data-sourcepos="1:1-1:9" dir="auto"><strong>foo bar</strong></p> - wysiwyg: |- + wysiwyg: | <p><strong>foo bar</strong></p> ``` @@ -684,8 +762,8 @@ depending on how the implementations evolve. #### `spec/fixtures/glfm/example_snapshots/prosemirror_json.yml` -`spec/fixtures/glfm/example_snapshots/prosemirror_json.yml` contains the ProseMirror -JSON for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` +[`spec/fixtures/glfm/example_snapshots/prosemirror_json.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/fixtures/glfm/example_snapshots/prosemirror_json.yml) +contains the ProseMirror JSON for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` - It is generated (or updated) from the frontend code via the `update-example-snapshots.rb` script, but can be manually updated for examples with incomplete implementations. @@ -715,3 +793,28 @@ JSON for each entry in `spec/fixtures/glfm/example_snapshots/examples_index.yml` ] } ``` + +## Workflows + +This section describes how the scripts can be used to manage the GLFM specification and tests. + +### Update the GLFM specification and run conformance tests + +1. Run [`update-specification.rb`](#update-specificationrb-script) to update the GLFM specification [output specification files](#output-specification-files). +1. Visually inspect and confirm any resulting changes to the [output specification files](#output-specification-files). +1. Run [`run-spec-tests.sh`](http://gdk.test:3005/ee/development/gitlab_flavored_markdown/specification_guide/index.html#run-spec-testssh-script) to run the conformance tests against the canonicalized GLFM specification. +1. Commit any changes to the [output specification files](#output-specification-files). + +### Update the example snapshots and run snapshot tests + +1. If you are working on an in-progress feature or bug, make any necessary manual updates to the [input specification files](#input-specification-files). This may include: + 1. Updating the canonical Markdown or HTML examples in `glfm_specification/input/gitlab_flavored_markdown/glfm_canonical_examples.txt`. + 1. Updating `glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml` to reflect the current status of the examples or tests. +1. Run [`update-specification.rb`](#update-specificationrb-script) to update the `spec.txt` to reflect any changes which were made to the [input specification files](#input-specification-files). +1. Visually inspect and confirm any resulting changes to the [output specification files](#output-specification-files). +1. Run [`update-example-snapshots.rb`](#update-example-snapshotsrb-script) to update the [example snapshot files](#example-snapshot-files). +1. Visually inspect and confirm any resulting changes to the [example snapshot files](#example-snapshot-files). +1. Run [`run-snapshot-tests.sh`](#run-snapshot-testssh-script) as a convenience script to run all relevant frontend (RSpec) and backend (Jest) tests which use the example snapshots. + 1. Any frontend or backend snapshot test may also be run individually. + 1. All frontend and backend tests are also run as part of the continuous integration suite, as they normally are. +1. Commit any changes to the [input specification files](#input-specification-files), [output specification files](#output-specification-files), or [example snapshot files](#example-snapshot-files). diff --git a/doc/development/go_guide/dependencies.md b/doc/development/go_guide/dependencies.md index 8aa8f286edc..0c2ce4f2b48 100644 --- a/doc/development/go_guide/dependencies.md +++ b/doc/development/go_guide/dependencies.md @@ -102,7 +102,7 @@ malicious party without causing build failures. Go 1.12+ can be configured to use a checksum database. If configured to do so, when Go fetches a dependency and there is no corresponding entry in `go.sum`, Go -queries the configured checksum database(s) for the checksum of the +queries the configured checksum databases for the checksum of the dependency instead of calculating it from the downloaded dependency. If the dependency cannot be found in the checksum database, the build fails. If the downloaded dependency's checksum does not match the result from the checksum diff --git a/doc/development/go_guide/go_upgrade.md b/doc/development/go_guide/go_upgrade.md index 3267d1262f0..4e2a0d95910 100644 --- a/doc/development/go_guide/go_upgrade.md +++ b/doc/development/go_guide/go_upgrade.md @@ -158,7 +158,7 @@ if you need help finding the correct person or labels: | GitLab Quality Images | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab-build-images/-/issues) | | GitLab Shell | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab-shell/-/issues) | | GitLab Workhorse | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) | -| Labkit | [Issue Tracker](https://gitlab.com/gitlab-org/labkit/-/issues) | +| LabKit | [Issue Tracker](https://gitlab.com/gitlab-org/labkit/-/issues) | | [Node Exporter](https://github.com/prometheus/node_exporter) | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) | | [PgBouncer Exporter](https://github.com/prometheus-community/pgbouncer_exporter) | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) | | [Postgres Exporter](https://github.com/prometheus-community/postgres_exporter) | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) | diff --git a/doc/development/index.md b/doc/development/index.md index 5c0cc7f9718..3d5ec24d3e2 100644 --- a/doc/development/index.md +++ b/doc/development/index.md @@ -21,7 +21,7 @@ For information on using GitLab to work on your own software projects, see the For information on working with the GitLab APIs, see the [API documentation](../api/index.md). For information about how to install, configure, update, and upgrade your own -GitLab instance, see the [administration documentation](../administration/index.md). +GitLab instance, see the [Administrator documentation](../administration/index.md). ## Get started @@ -144,7 +144,7 @@ In these cases, use the following workflow: If the page is not assigned to a specific group, follow the [Technical Writing review process for development guidelines](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines). The Technical Writer may ask for additional approvals as previously suggested before merging the MR. - + ### Reviewer values > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57293) in GitLab 14.1. diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index 34ac307c98a..e595fea6d96 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -134,7 +134,7 @@ By default, the integration form provides: - Checkboxes for each of the trigger events returned from `Integration#configurable_events`. You can also add help text at the top of the form by either overriding `Integration#help`, -or providing a template in `app/views/projects/services/$INTEGRATION_NAME/_help.html.haml`. +or providing a template in `app/views/shared/integrations/$INTEGRATION_NAME/_help.html.haml`. To add your custom properties to the form, you can define the metadata for them in `Integration#fields`. @@ -275,7 +275,7 @@ as described above in [Customize the frontend form](#customize-the-frontend-form our [usability guidelines](https://design.gitlab.com/usability/helping-users) for help text. For more detailed documentation, provide a page in `doc/user/project/integrations`, -and link it from the [Integrations overview](../../user/project/integrations/overview.md). +and link it from the [Integrations overview](../../user/project/integrations/index.md). You can also refer to our general [documentation guidelines](../documentation/index.md). diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 5f7cccdab64..0f4fa1a97a8 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -290,9 +290,6 @@ useful when debugging. The default value for `SECURE_LOG_LEVEL` should be set to `info`. When executing command lines, scanners should use the `debug` level to log the command line and its output. -For instance, the [bundler-audit](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) scanner -uses the `debug` level to log the command line `bundle audit check --quiet`, -and what `bundle audit` writes to the standard output. If the command line fails, then it should be logged with the `error` log level; this makes it possible to debug the problem without having to change the log level to `debug` and rerun the scanning job. @@ -679,7 +676,7 @@ The confidence ranges from `Low` to `Confirmed`, but it can also be `Unknown`, Valid values are: `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, or `Confirmed` `Unknown` values means that data is unavailable to determine it's actual value. Therefore, it may be `high`, `medium`, or `low`, -and needs to be investigated. We have [provided a chart](../../user/application_security/sast/analyzers.md#analyzers-data) +and needs to be investigated. We have [provided a chart](../../user/application_security/sast/analyzers.md#data-provided-by-analyzers) of the available SAST Analyzers and what data is currently available. #### Remediations diff --git a/doc/development/internal_api/index.md b/doc/development/internal_api/index.md index cdbc674e0a5..dca71413564 100644 --- a/doc/development/internal_api/index.md +++ b/doc/development/internal_api/index.md @@ -478,28 +478,6 @@ curl --request POST --header "Gitlab-Kas-Api-Request: <JWT token>" --header "Con --data '{"gitops_sync_count":1}' "http://localhost:3000/api/v4/internal/kubernetes/usage_metrics" ``` -### GitLab agent alert metrics - -Called from GitLab agent server (KAS) to save alerts derived from Cilium on Kubernetes -Cluster. - -| Attribute | Type | Required | Description | -|:----------|:-------|:---------|:------------| -| `alert` | Hash | yes | Alerts detail. Same format as [3rd party alert](../../operations/incident_management/integrations.md#customize-the-alert-payload-outside-of-gitlab). | - -```plaintext -POST internal/kubernetes/modules/cilium_alert -``` - -Example Request: - -```shell -curl --request POST --header "Gitlab-Kas-Api-Request: <JWT token>" \ - --header "Authorization: Bearer <agent token>" --header "Content-Type: application/json" \ - --data '"{\"alert\":{\"title\":\"minimal\",\"message\":\"network problem\",\"evalMatches\":[{\"value\":1,\"metric\":\"Count\",\"tags\":{}}]}}"' \ - "http://localhost:3000/api/v4/internal/kubernetes/modules/cilium_alert" -``` - ### Create Starboard vulnerability Called from the GitLab agent server (`kas`) to create a security vulnerability diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index a6d9c754838..ee261769d82 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -54,7 +54,7 @@ webserver, and can lead to a denial-of-service (DoS) attack in GitLab as the Kubernetes cluster response times are outside of our control. The easiest way to ensure your calls happen a background process is to -delegate any such work to happen in a [Sidekiq worker](sidekiq_style_guide.md). +delegate any such work to happen in a [Sidekiq worker](sidekiq/index.md). You may want to make calls to Kubernetes and return the response, but a background worker isn't a good fit. Consider using diff --git a/doc/development/maintenance_mode.md b/doc/development/maintenance_mode.md index c2fd4bab605..a118d9cf0ad 100644 --- a/doc/development/maintenance_mode.md +++ b/doc/development/maintenance_mode.md @@ -11,8 +11,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab Maintenance Mode **only** blocks writes from HTTP and SSH requests at the application level in a few key places within the rails application. [Search the codebase for `maintenance_mode?`.](https://gitlab.com/search?search=maintenance_mode%3F&group_id=9970&project_id=278964&scope=blobs&search_code=false&snippets=false&repository_ref=) -- [the read-only database method](https://gitlab.com/gitlab-org/gitlab/-/blob/2425e9de50c678413ceaad6ee3bf66f42b7e228c/ee/lib/ee/gitlab/database.rb#L13), which toggles special behavior when we are not allowed to write to the database. [Search the codebase for `Gitlab::Database.read_only?`.](https://gitlab.com/search?search=Gitlab%3A%3ADatabase.read_only%3F&group_id=9970&project_id=278964&scope=blobs&search_code=false&snippets=false&repository_ref=) -- [the read-only middleware](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/middleware/read_only/controller.rb), where HTTP requests that cause database writes are blocked, unless explicitly allowed. +- [the read-only database method](https://gitlab.com/gitlab-org/gitlab/-/blob/2425e9de50c678413ceaad6ee3bf66f42b7e228c/ee/lib/ee/gitlab/database.rb#L13), which toggles special behavior when we are not allowed to write to the database. We use this method for possible places where writes could occur in GET requests. [Search the codebase for `Gitlab::Database.read_only?`.](https://gitlab.com/search?search=Gitlab%3A%3ADatabase.read_only%3F&group_id=9970&project_id=278964&scope=blobs&search_code=false&snippets=false&repository_ref=) +- [the read-only middleware](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/gitlab/middleware/read_only/controller.rb), where HTTP requests that cause database writes are blocked, unless explicitly allowed (for example, GET requests). - [Git push access via SSH is denied](https://gitlab.com/gitlab-org/gitlab/-/blob/2425e9de50c678413ceaad6ee3bf66f42b7e228c/ee/lib/ee/gitlab/git_access.rb#L13) by returning 401 when `gitlab-shell` POSTs to [`/internal/allowed`](internal_api/index.md) to [check if access is allowed](internal_api/index.md#git-authentication). - [Container registry authentication service](https://gitlab.com/gitlab-org/gitlab/-/blob/2425e9de50c678413ceaad6ee3bf66f42b7e228c/ee/app/services/ee/auth/container_registry_authentication_service.rb#L12), where updates to the container registry are blocked. diff --git a/doc/development/merge_request_application_and_rate_limit_guidelines.md b/doc/development/merge_request_application_and_rate_limit_guidelines.md index 94ae126802a..62bf62f6275 100644 --- a/doc/development/merge_request_application_and_rate_limit_guidelines.md +++ b/doc/development/merge_request_application_and_rate_limit_guidelines.md @@ -14,7 +14,7 @@ Every new feature should have safe usage limits included in its implementation. Limits are applicable for: - System-level resource pools such as API requests, SSHD connections, database connections, storage, and so on. -- Domain-level objects such as CI minutes, groups, sign-in attempts, and so on. +- Domain-level objects such as CI/CD minutes, groups, sign-in attempts, and so on. ## When limits are required diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index fe8e730d64e..5e7fe9cc8fb 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -206,7 +206,7 @@ By default, this `Gitlab::SQL::CTE` class forces materialization through adding (this behavior is implemented using a custom Arel node `Gitlab::Database::AsWithMaterialized` under the surface). WARNING: -We plan to drop the support for PostgreSQL 11. Upgrading to GitLab 14.0 requires PostgreSQL 12 or higher. +Upgrading to GitLab 14.0 requires PostgreSQL 12 or higher. ## Cached Queries @@ -526,7 +526,7 @@ end The usage of shared temporary storage is required if your intent is to persistent file for a disk-based storage, and not Object Storage. -[Workhorse direct_upload](uploads/implementation.md#direct-upload) when accepting file +[Workhorse direct_upload](uploads/index.md#direct-upload) when accepting file can write it to shared storage, and later GitLab Rails can perform a move operation. The move operation on the same destination is instantaneous. The system instead of performing `copy` operation just re-attaches file into a new place. @@ -550,7 +550,7 @@ that implements a seamless support for Shared and Object Storage-based persisten #### Data access Each feature that accepts data uploads or allows to download them needs to use -[Workhorse direct_upload](uploads/implementation.md#direct-upload). It means that uploads needs to be +[Workhorse direct_upload](uploads/index.md#direct-upload). It means that uploads needs to be saved directly to Object Storage by Workhorse, and all downloads needs to be served by Workhorse. @@ -562,5 +562,5 @@ can time out, which is especially problematic for slow clients. If clients take to upload/download the processing slot might be killed due to request processing timeout (usually between 30s-60s). -For the above reasons it is required that [Workhorse direct_upload](uploads/implementation.md#direct-upload) is implemented +For the above reasons it is required that [Workhorse direct_upload](uploads/index.md#direct-upload) is implemented for all file uploads and downloads. diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 086e061452b..aebecd90574 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -110,6 +110,11 @@ table, that column is added at the bottom. Please do not reorder columns manually for existing tables as this causes confusion to other people using `db/structure.sql` generated by Rails. +NOTE: +[Creating an index asynchronously requires two merge requests.](adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously) +When done, commit the schema change in the merge request +that adds the index with `add_concurrent_index`. + When your local database in your GDK is diverging from the schema from `main` it might be hard to cleanly commit the schema changes to Git. In that case you can use the `scripts/regenerate-schema` script to @@ -127,6 +132,24 @@ scripts/regenerate-schema TARGET=12-9-stable-ee scripts/regenerate-schema ``` +The `scripts/regenerate-schema` script can create additional differences. +If this happens, use a manual procedure where `<migration ID>` is the `DATETIME` +part of the migration file. + +```shell +# Rebase against master +git rebase master + +# Rollback changes +VERSION=<migration ID> bundle exec rails db:rollback:main + +# Checkout db/structure.sql from master +git checkout origin/master db/structure.sql + +# Migrate changes +VERSION=<migration ID> bundle exec rails db:migrate:main +``` + ## Avoiding downtime The document ["Avoiding downtime in migrations"](database/avoiding_downtime_in_migrations.md) specifies @@ -487,7 +510,7 @@ end ### When to use the helper method You can **only** use the `with_lock_retries` helper method when the execution is not already inside -an open transaction (using Postgres subtransactions is discouraged). It can be used with +an open transaction (using PostgreSQL subtransactions is discouraged). It can be used with standard Rails migration helper methods. Calling more than one migration helper is not a problem if they're executed on the same table. @@ -602,7 +625,7 @@ end ``` You must explicitly name indexes that are created with more complex -definitions beyond table name, column name(s) and uniqueness constraint. +definitions beyond table name, column names, and uniqueness constraint. Consult the [Adding Database Indexes](adding_database_indexes.md#requirements-for-naming-indexes) guide for more details. diff --git a/doc/development/new_fe_guide/development/performance.md b/doc/development/new_fe_guide/development/performance.md index f34c407da84..ee853942cb9 100644 --- a/doc/development/new_fe_guide/development/performance.md +++ b/doc/development/new_fe_guide/development/performance.md @@ -8,15 +8,15 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Monitoring -We have a performance dashboard available in one of our [Grafana instances](https://dashboards.gitlab.net/d/1EBTz3Dmz/sitespeed-page-summary?orgId=1). This dashboard automatically aggregates metric data from [sitespeed.io](https://www.sitespeed.io/) every 6 hours. These changes are displayed after a set number of pages are aggregated. +We have a performance dashboard available in one of our [Grafana instances](https://dashboards.gitlab.net/d/000000043/sitespeed-page-summary?orgId=1). This dashboard automatically aggregates metric data from [sitespeed.io](https://www.sitespeed.io/) every 4 hours. These changes are displayed after a set number of pages are aggregated. -These pages can be found inside a text file in the [`gitlab-build-images` repository](https://gitlab.com/gitlab-org/gitlab-build-images) called [`gitlab.txt`](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/scripts/gitlab.txt) -Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing URLs of pages from this text file. Please have a [frontend monitoring expert](https://about.gitlab.com/company/team/) review your changes before assigning to a maintainer of the `gitlab-build-images` project. The changes are pushed live on the next scheduled run after the changes are merged into `main`. +These pages can be found inside text files in the [`sitespeed-measurement-setup` repository](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup) called [`gitlab`](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup/-/tree/master/gitlab) +Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing URLs of pages to the text files. The changes are pushed live on the next scheduled run after the changes are merged into `main`. -There are 3 recommended high impact metrics to review on each page: +There are 3 recommended high impact metrics (core web vitals) to review on each page: -- [First visual change](https://web.dev/first-meaningful-paint/) -- [Speed Index](https://github.com/WPO-Foundation/webpagetest-docs/blob/master/user/Metrics/SpeedIndex.md) -- [Visual Complete 95%](https://github.com/WPO-Foundation/webpagetest-docs/blob/master/user/Metrics/SpeedIndex.md) +- [Largest Contentful Paint](https://web.dev/lcp/) +- [First Input Delay](https://web.dev/fid/) +- [Cumulative Layout Shift](https://web.dev/cls/) For these metrics, lower numbers are better as it means that the website is more performant. diff --git a/doc/development/new_fe_guide/modules/widget_extensions.md b/doc/development/new_fe_guide/modules/widget_extensions.md index 638a0a2a85b..d3be8981abb 100644 --- a/doc/development/new_fe_guide/modules/widget_extensions.md +++ b/doc/development/new_fe_guide/modules/widget_extensions.md @@ -36,6 +36,7 @@ export default { }, expandEvent: '', // Optional: RedisHLL event name to track expanding content enablePolling: false, // Optional: Tells extension to poll for data + modalComponent: null, // Optional: The component to use for the modal computed: { summary(data) {}, // Required: Level 1 summary text statusIcon(data) {}, // Required: Level 1 status icon @@ -128,6 +129,14 @@ mentioned below: text: '', // Required: Text to be displayed inside badge variant: '', // Optional: GitLab UI badge variant, defaults to info }, + link: { // Optional: Link to a URL displayed after text + text: '', // Required: Text of the link + href: '', // Optional: URL for the link + }, + modal: { // Optional: Link to open a modal displayed after text + text: '', // Required: Text of the link + onClick: () => {} // Optional: Function to run when link is clicked, i.e. to set this.modalData + } actions: [], // Optional: Action button for row children: [], // Optional: Child content to render, structure matches the same structure } diff --git a/doc/development/packages.md b/doc/development/packages.md index 35a93c77c7f..6526bdd45a1 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -151,7 +151,7 @@ During this phase, the idea is to collect as much information as possible about 1. Empty file structure (API file, base service for this package) 1. Authentication system for "logging in" to the package manager 1. Identify metadata and create applicable tables - 1. Workhorse route for [object storage direct upload](uploads/implementation.md#direct-upload) + 1. Workhorse route for [object storage direct upload](uploads/index.md#direct-upload) 1. Endpoints required for upload/publish 1. Endpoints required for install/download 1. Endpoints required for required actions @@ -210,7 +210,7 @@ File uploads should be handled by GitLab Workhorse using object accelerated uplo the workhorse proxy that checks all incoming requests to GitLab intercept the upload request, upload the file, and forward a request to the main GitLab codebase only containing the metadata and file location rather than the file itself. An overview of this process can be found in the -[development documentation](uploads/implementation.md#direct-upload). +[development documentation](uploads/index.md#direct-upload). In terms of code, this means a route must be added to the [GitLab Workhorse project](https://gitlab.com/gitlab-org/gitlab-workhorse) for each upload endpoint being added diff --git a/doc/development/performance.md b/doc/development/performance.md index 1e3e0570206..6d0b833a2da 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -75,7 +75,6 @@ GitLab provides built-in tools to help improve performance and availability: - [Profiling](profiling.md). - [Distributed Tracing](distributed_tracing.md) - [GitLab Performance Monitoring](../administration/monitoring/performance/index.md). -- [Request Profiling](../administration/monitoring/performance/request_profiling.md). - [QueryRecoder](query_recorder.md) for preventing `N+1` regressions. - [Chaos endpoints](chaos_endpoints.md) for testing failure scenarios. Intended mainly for testing availability. - [Service measurement](service_measurement.md) for measuring and logging service execution. @@ -319,7 +318,7 @@ You can do this when using the [performance bar](profiling.md#speedscope-flamegr and when [profiling code blocks](https://github.com/jlfwong/speedscope/wiki/Importing-from-stackprof-(ruby)). This option isn't supported by `bin/rspec-stackprof`. -You can profile speciific methods by using `--method method_name`: +You can profile specific methods by using `--method method_name`: ```shell $ stackprof tmp/project_policy_spec.rb.dump --method access_allowed_to diff --git a/doc/development/permissions.md b/doc/development/permissions.md index 47aebc2f4d2..f3818e92fec 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -86,10 +86,10 @@ module): - Maintainer (`40`) - Owner (`50`) -If a user is the member of both a project and the project parent group(s), the +If a user is the member of both a project and the project parent groups, the higher permission is taken into account for the project. -If a user is the member of a project, but not the parent group(s), they +If a user is the member of a project, but not the parent groups, they can still view the groups and their entities (like epics). Project membership (where the group membership is already taken into account) diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index e0b236bc5fc..b70f07ea7d9 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -12,7 +12,7 @@ which itself includes files under [`.gitlab/ci/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/.gitlab/ci) for easier maintenance. -We're striving to [dogfood](https://about.gitlab.com/handbook/engineering/principles/#dogfooding) +We're striving to [dogfood](https://about.gitlab.com/handbook/engineering/development/principles/#dogfooding) GitLab [CI/CD features and best-practices](../ci/yaml/index.md) as much as possible. @@ -53,7 +53,7 @@ In summary: To identify the minimal set of tests needed, we use the [`test_file_finder` gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder), with two strategies: -- dynamic mapping from test coverage tracing (generated via the [Crystalball gem](https://github.com/toptal/crystalball)) +- dynamic mapping from test coverage tracing (generated via the [`Crystalball` gem](https://github.com/toptal/crystalball)) ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/47d507c93779675d73a05002e2ec9c3c467cd698/tooling/bin/find_tests#L15)) - static mapping maintained in the [`tests.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tests.yml) for special cases that cannot be mapped via coverage tracing ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/47d507c93779675d73a05002e2ec9c3c467cd698/tooling/bin/find_tests#L12)) @@ -93,7 +93,7 @@ In addition, there are a few circumstances where we would always run the full Je ### Fork pipelines We only run the minimal RSpec & Jest jobs for fork pipelines unless the `pipeline:run-all-rspec` -label is set on the MR. The goal is to reduce the CI minutes consumed by fork pipelines. +label is set on the MR. The goal is to reduce the CI/CD minutes consumed by fork pipelines. See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1170). @@ -160,7 +160,7 @@ Our current RSpec tests parallelization setup is as follows: `knapsack/report-master.json` file: - The `knapsack/report-master.json` file is fetched from the latest `main` pipeline which runs `update-tests-metadata` (for now it's the 2-hourly scheduled master pipeline), if it's not here we initialize the file with `{}`. -1. Each `[rspec|rspec-ee] [unit|integration|system|geo] n m` job are run with +1. Each `[rspec|rspec-ee] [migration|unit|integration|system|geo] n m` job are run with `knapsack rspec` and should have an evenly distributed share of tests: - It works because the jobs have access to the `knapsack/report-master.json` since the "artifacts from all previous stages are passed by default". @@ -170,7 +170,7 @@ Our current RSpec tests parallelization setup is as follows: `Report specs`, not under `Leftover specs`. 1. The `update-tests-metadata` job (which only runs on scheduled pipelines for [the canonical project](https://gitlab.com/gitlab-org/gitlab) takes all the - `knapsack/rspec*_pg_*.json` files and merge them all together into a single + `knapsack/rspec*.json` files and merge them all together into a single `knapsack/report-master.json` file that is saved as artifact. After that, the next pipeline uses the up-to-date `knapsack/report-master.json` file. @@ -247,12 +247,18 @@ The `* as-if-jh` jobs are run in addition to the regular EE-context jobs. The `j The intent is to ensure that a change doesn't introduce a failure after `gitlab-org/gitlab` is synced to [GitLab JH](https://jihulab.com/gitlab-cn/gitlab). +### When to consider applying `pipeline:run-as-if-jh` label + +If a Ruby file is renamed and there's a corresponding [`prepend_mod` line](jh_features_review.md#jh-features-based-on-ce-or-ee-features), +it's likely that GitLab JH is relying on it and requires a corresponding +change to rename the module or class it's prepending. + ### Corresponding JH branch You can create a corresponding JH branch on [GitLab JH](https://jihulab.com/gitlab-cn/gitlab) by appending `-jh` to the branch name. If a corresponding JH branch is found, `* as-if-jh` jobs grab the `jh` folder from the respective branch, -rather than from the default branch. +rather than from the default branch `main-jh`. NOTE: For now, CI will try to fetch the branch on the [GitLab JH mirror](https://gitlab.com/gitlab-org/gitlab-jh-mirrors/gitlab), so it might take some time for the new JH branch to propagate to the mirror. @@ -345,7 +351,7 @@ We use the [`rules:`](../ci/yaml/index.md#rules) and [`needs:`](../ci/yaml/index to determine the jobs that need to be run in a pipeline. Note that an MR that includes multiple types of changes would have a pipelines that include jobs from multiple types (for example, a combination of docs-only and code-only pipelines). -Following are graphs of the critical paths for each pipeline type. Jobs that aren't part of the critical path are ommitted. +Following are graphs of the critical paths for each pipeline type. Jobs that aren't part of the critical path are omitted. ### Documentation pipeline @@ -508,12 +514,9 @@ The current stages are: - `test`: This stage includes most of the tests, and DB/migration jobs. - `post-test`: This stage includes jobs that build reports or gather data from the `test` stage's jobs (for example, coverage, Knapsack metadata, and so on). -- `review-prepare`: This stage includes a job that build the CNG images that are - later used by the (Helm) Review App deployment (see - [Review Apps](testing_guide/review_apps.md) for details). -- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps. -- `dast`: This stage includes jobs that run a DAST full scan against the Review App -that is deployed in stage `review`. +- `review`: This stage includes jobs that build the CNG images, deploy them, and + run end-to-end tests against Review Apps (see [Review Apps](testing_guide/review_apps.md) for details). + It also includes Docs Review App jobs. - `qa`: This stage includes jobs that perform QA tasks against the Review App that is deployed in stage `review`. - `post-qa`: This stage includes jobs that build reports or gather data from diff --git a/doc/development/product_qualified_lead_guide/index.md b/doc/development/product_qualified_lead_guide/index.md index 2395689ada2..dcd8b33e5c5 100644 --- a/doc/development/product_qualified_lead_guide/index.md +++ b/doc/development/product_qualified_lead_guide/index.md @@ -16,8 +16,8 @@ A hand-raise PQL is a user who requests to speak to sales from within the produc 1. Set up CustomersDot to talk to a staging instance of Platypus. 1. Set up CustomersDot using the [normal install instructions](https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/doc/setup/installation_steps.md). -1. Set the `CUSTOMER_PORTAL_URL` env var to your local (or ngrok) URL of your CustomersDot instance. -1. Place `export CUSTOMER_PORTAL_URL='https://XXX.ngrok.io/'` in your shell rc script (~/.zshrc or ~/.bash_profile or ~/.bashrc) and restart GDK. +1. Set the `CUSTOMER_PORTAL_URL` environment variable to your local (or ngrok) URL of your CustomersDot instance. +1. Place `export CUSTOMER_PORTAL_URL='https://XXX.ngrok.io/'` in your shell rc script (`~/.zshrc` or `~/.bash_profile` or `~/.bashrc`) and restart GDK. 1. Enter the credentials on CustomersDot development to Platypus in your `/config/secrets.yml` and restart. Credentials for the Platypus Staging are in the 1Password Growth vault. The URL for staging is `https://staging.ci.nexus.gitlabenvironment.cloud`. ```yaml diff --git a/doc/development/python_guide/index.md b/doc/development/python_guide/index.md index fe5492c3bd8..77dd328b513 100644 --- a/doc/development/python_guide/index.md +++ b/doc/development/python_guide/index.md @@ -25,6 +25,18 @@ To install `pyenv` on macOS, you can use [Homebrew](https://brew.sh/) with: brew install pyenv ``` +### Windows + +`pyenv` does not officially support Windows and does not work in Windows outside the Windows Subsystem for Linux. If you are a Windows user, you can use `pyenv-win`. + +To install `pyenv-win` on Windows, run the following PowerShell command: + +```shell +Invoke-WebRequest -UseBasicParsing -Uri "https://raw.githubusercontent.com/pyenv-win/pyenv-win/master/pyenv-win/install-pyenv-win.ps1" -OutFile "./install-pyenv-win.ps1"; &"./install-pyenv-win.ps1" +``` + +[Learn more about `pyenv-win`](https://github.com/pyenv-win/pyenv-win). + ### Linux To install `pyenv` on Linux, you can run the command below: diff --git a/doc/development/rails_initializers.md b/doc/development/rails_initializers.md index ee73dac2b72..9bf4109f1cb 100644 --- a/doc/development/rails_initializers.md +++ b/doc/development/rails_initializers.md @@ -20,3 +20,21 @@ Some examples where you would need to do this are: 1. Modifying Rails' `config.autoload_paths` 1. Changing configuration that Zeitwerk uses, for example, inflections + +## Database connections in initializers + +Ideally, database connections are not opened from Rails initializers. Opening a +database connection (e.g. checking the database exists, or making a database +query) from an initializer means that tasks like `db:drop`, and +`db:test:prepare` will fail because an active session prevents the database from +being dropped. + +To help detect when database connections are opened from initializers, we now +warn in stderr. For example: + +```shell +DEPRECATION WARNING: Database connection should not be called during initializers (called from block in <module:HasVariable> at app/models/concerns/ci/has_variable.rb:22) +``` + +If you wish to print out the full backtrace, set the +`DEBUG_INITIALIZER_CONNECTIONS` environment variable. diff --git a/doc/development/rails_update.md b/doc/development/rails_update.md index 1a30e606c17..8999ac90f4c 100644 --- a/doc/development/rails_update.md +++ b/doc/development/rails_update.md @@ -88,7 +88,7 @@ To efficiently and quickly find which Rails change caused the spec failure you c For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. Replace `<NEW_VERSION_TAG>` with the tag where the spec is red and `<OLD_VERSION_TAG>` with the one with the green spec. For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. In the output, you can see how many steps approximately it takes to find the commit. -1. Start the `git bisect` process and pass spec's file name(s) to `scripts/rails-update-bisect` as an argument or arguments. It can be faster to pick only one example instead of an entire spec file. +1. Start the `git bisect` process and pass spec's filenames to `scripts/rails-update-bisect` as arguments. It can be faster to pick only one example instead of an entire spec file. ```shell git bisect run <GDK_FOLDER>/gitlab/scripts/rails-update-bisect spec/models/ability_spec.rb diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 1e9367ecee4..0538add59b5 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -366,8 +366,8 @@ The docs generator code comes from our side giving us more flexibility, like usi To edit the content, you may need to edit the following: -- The template. You can edit the template at `lib/gitlab/graphql/docs/templates/default.md.haml`. - The actual renderer is at `Gitlab::Graphql::Docs::Renderer`. +- The template. You can edit the template at `tooling/graphql/docs/templates/default.md.haml`. + The actual renderer is at `Tooling::Graphql::Docs::Renderer`. - The applicable `description` field in the code, which [Updates machine-readable schema files](#update-machine-readable-schema-files), which is then used by the `rake` task described earlier. diff --git a/doc/development/redis.md b/doc/development/redis.md index 75170b8c746..d5f526f2d32 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab uses [Redis](https://redis.io) for the following distinct purposes: - Caching (mostly via `Rails.cache`). -- As a job processing queue with [Sidekiq](sidekiq_style_guide.md). +- As a job processing queue with [Sidekiq](sidekiq/index.md). - To manage the shared application state. - To store CI trace chunks. - As a Pub/Sub queue backend for ActionCable. @@ -147,12 +147,11 @@ mostly for fine-grained control of Redis usage, so they wouldn't be used in combination with the `Rails.cache` wrapper: we'd either use `Rails.cache` or these classes and literal Redis commands. -`Rails.cache` or these classes and literal Redis commands. We prefer -using `Rails.cache` so we can reap the benefits of future optimizations -done to Rails. It is worth noting that Ruby objects are +We prefer using `Rails.cache` so we can reap the benefits of future +optimizations done to Rails. Ruby objects are [marshalled](https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache/redis_cache_store.rb#L447) -when written to Redis, so we need to pay attention to not to store huge -objects, or untrusted user input. +when written to Redis, so we must pay attention to store neither huge objects, +nor untrusted user input. Typically we would only use these classes when at least one of the following is true: diff --git a/doc/development/redis/new_redis_instance.md b/doc/development/redis/new_redis_instance.md index 96f860f3890..389cddbb4e5 100644 --- a/doc/development/redis/new_redis_instance.md +++ b/doc/development/redis/new_redis_instance.md @@ -232,8 +232,8 @@ a developer will need to add an implementation for missing Redis commands before | metrics name | type | labels | description | |-------------------------------------------------|--------------------|------------------------|----------------------------------------------------| -| gitlab_redis_multi_store_read_fallback_total | Prometheus Counter | command, instance_name | Client side Redis MultiStore reading fallback total| -| gitlab_redis_multi_store_method_missing_total | Prometheus Counter | command, instance_name | Client side Redis MultiStore method missing total | +| `gitlab_redis_multi_store_read_fallback_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore reading fallback total| +| `gitlab_redis_multi_store_method_missing_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore method missing total | ## Step 4: clean up after the migration diff --git a/doc/development/routing.md b/doc/development/routing.md index 8fca9b00157..41961c2288f 100644 --- a/doc/development/routing.md +++ b/doc/development/routing.md @@ -31,6 +31,16 @@ we introduced the `/-/` scope. The purpose of it is to separate group or project paths from the rest of the routes. Also it helps to reduce the number of [reserved names](../user/reserved_names.md). +## View all available routes + +You can view and find routes from the console by running: + +```shell +rails routes | grep crm +``` + +You can also view routes in your browser by going to [http://gdk.test:3000/rails/info/routes](http://gdk.test:3000/rails/info/routes). + ## Global routes We have a number of global routes. For example: diff --git a/doc/development/ruby_upgrade.md b/doc/development/ruby_upgrade.md index a208a93e300..3b89a6fd1ea 100644 --- a/doc/development/ruby_upgrade.md +++ b/doc/development/ruby_upgrade.md @@ -144,7 +144,7 @@ A [build matrix definition](../ci/yaml/index.md#parallelmatrix) can do this effi When upgrading Ruby, consider updating the following repositories: - [Gitaly](https://gitlab.com/gitlab-org/gitaly) ([example](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3771)) -- [GitLab Labkit](https://gitlab.com/gitlab-org/labkit-ruby) ([example](https://gitlab.com/gitlab-org/labkit-ruby/-/merge_requests/79)) +- [GitLab LabKit](https://gitlab.com/gitlab-org/labkit-ruby) ([example](https://gitlab.com/gitlab-org/labkit-ruby/-/merge_requests/79)) - [GitLab Exporter](https://gitlab.com/gitlab-org/gitlab-exporter) ([example](https://gitlab.com/gitlab-org/gitlab-exporter/-/merge_requests/150)) - [GitLab Experiment](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment) ([example](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment/-/merge_requests/128)) - [Gollum Lib](https://gitlab.com/gitlab-org/gollum-lib) ([example](https://gitlab.com/gitlab-org/gollum-lib/-/merge_requests/21)) @@ -272,4 +272,4 @@ and merged back independently. - **Give yourself enough time to fix problems ahead of a milestone release.** GitLab moves fast. As a Ruby upgrade requires many MRs to be sent and reviewed, make sure all changes are merged at least a week before the 22nd. This gives us extra time to act if something breaks. If in doubt, it is better to -postpone the upgrade to the following month, as we [prioritize availability over velocity](https://about.gitlab.com/handbook/engineering/principles/#prioritizing-technical-decisions). +postpone the upgrade to the following month, as we [prioritize availability over velocity](https://about.gitlab.com/handbook/engineering/development/principles/#prioritizing-technical-decisions). diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 8a86a46d1d3..3e46891d20e 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -461,7 +461,9 @@ References: ### Description -Path Traversal vulnerabilities grant attackers access to arbitrary directories and files on the server that is executing an application, including data, code or credentials. +Path Traversal vulnerabilities grant attackers access to arbitrary directories and files on the server that is executing an application. This data can include data, code or credentials. + +Traversal can occur when a path includes directories. A typical malicious example includes one or more `../`, which tells the file system to look in the parent directory. Supplying many of them in a path, for example `../../../../../../../etc/passwd`, usually resolves to `/etc/passwd`. If the file system is instructed to look back to the root directory and can't go back any further, then extra `../` are ignored. The file system then looks from the root, resulting in `/etc/passwd` - a file you definitely do not want exposed to a malicious attacker! ### Impact @@ -510,6 +512,44 @@ requires :file_path, type: String, file_path: true Absolute paths are not allowed by default. If allowing an absolute path is required, you need to provide an array of paths to the parameter `allowlist`. +### Misleading behavior + +Some methods used to construct file paths can have non-intuitive behavior. To properly validate user input, be aware +of these behaviors. + +#### Ruby + +The Ruby method [`Pathname.join`](https://ruby-doc.org/stdlib-2.7.4/libdoc/pathname/rdoc/Pathname.html#method-i-join) +joins path names. Using methods in a specific way can result in a path name typically prohibited in +normal use. In the examples below, we see attempts to access `/etc/passwd`, which is a sensitive file: + +```ruby +require 'pathname' + +p = Pathname.new('tmp') +print(p.join('log', 'etc/passwd', 'foo')) +# => tmp/log/etc/passwd/foo +``` + +Assuming the second parameter is user-supplied and not validated, submitting a new absolute path +results in a different path: + +```ruby +print(p.join('log', '/etc/passwd', '')) +# renders the path to "/etc/passwd", which is not what we expect! +``` + +#### Golang + +Golang has similar behavior with [`path.Clean`](https://pkg.go.dev/path#example-Clean). Remember that with many file systems, using `../../../../` traverses up to the root directory. Any remaining `../` are ignored. This example may give an attacker access to `/etc/passwd`: + +```golang +path.Clean("/../../etc/passwd") +// renders the path to "etc/passwd"; the file path is relative to whatever the current directory is +path.Clean("../../etc/passwd") +// renders the path to "../../etc/passwd"; the file path will look back up to two parent directories! +``` + ## OS command injection guidelines Command injection is an issue in which an attacker is able to execute arbitrary commands on the host @@ -620,7 +660,7 @@ cfg := &tls.Config{ } ``` -For **Ruby**, you can use [HTTParty](https://github.com/jnunemaker/httparty) and specify TLS 1.3 version as well as ciphers: +For **Ruby**, you can use [`HTTParty`](https://github.com/jnunemaker/httparty) and specify TLS 1.3 version as well as ciphers: Whenever possible this example should be **avoided** for security purposes: @@ -665,7 +705,7 @@ tls.Config{ This example was taken [here](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/871b52dc700f1a66f6644fbb1e78a6d463a6ff83/internal/tool/tlstool/tlstool.go#L72). -For **Ruby**, you can use again [HTTParty](https://github.com/jnunemaker/httparty) and specify this time TLS 1.2 version alongside with the recommended ciphers: +For **Ruby**, you can use again [`HTTParty`](https://github.com/jnunemaker/httparty) and specify this time TLS 1.2 version alongside with the recommended ciphers: ```ruby response = GitLab::HTTP.perform_request(Net::HTTP::Get, 'https://gitlab.com', ssl_version: :TLSv1_2, ciphers: ['ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-CHACHA20-POLY1305', 'ECDHE-RSA-CHACHA20-POLY1305']) @@ -833,7 +873,7 @@ If a vulnerable application extracts an archive file with any of these file name #### Ruby -For zip files, the [rubyzip](https://rubygems.org/gems/rubyzip) Ruby gem is already patched against the Zip Slip vulnerability and will refuse to extract files that try to perform directory traversal, so for this vulnerable example we will extract a `tar.gz` file with `Gem::Package::TarReader`: +For zip files, the [`rubyzip`](https://rubygems.org/gems/rubyzip) Ruby gem is already patched against the Zip Slip vulnerability and will refuse to extract files that try to perform directory traversal, so for this vulnerable example we will extract a `tar.gz` file with `Gem::Package::TarReader`: ```ruby # Vulnerable tar.gz extraction example! @@ -1032,7 +1072,7 @@ Symlink attacks makes it possible for an attacker to read the contents of arbitr #### Ruby -For zip files, the [rubyzip](https://rubygems.org/gems/rubyzip) Ruby gem is already patched against symlink attacks as it simply ignores symbolic links, so for this vulnerable example we will extract a `tar.gz` file with `Gem::Package::TarReader`: +For zip files, the [`rubyzip`](https://rubygems.org/gems/rubyzip) Ruby gem is already patched against symlink attacks as it simply ignores symbolic links, so for this vulnerable example we will extract a `tar.gz` file with `Gem::Package::TarReader`: ```ruby # Vulnerable tar.gz extraction example! @@ -1210,3 +1250,36 @@ An example of well implemented `Gitlab::UrlBlocker.validate!` call that prevents ### Resources - [CWE-367: Time-of-check Time-of-use (TOCTOU) Race Condition](https://cwe.mitre.org/data/definitions/367.html) + +## Handling credentials + +Credentials can be: + +- Login details like username and password. +- Private keys. +- Tokens (PAT, runner tokens, JWT token, CSRF tokens, project access tokens, etc). +- Session cookies. +- Any other piece of information that can be used for authentication or authorization purposes. + +This sensitive data must be handled carefully to avoid leaks which could lead to unauthorized access. If you have questions or need help with any of the following guidance, talk to the GitLab AppSec team on Slack (`#sec-appsec`). + +### At rest + +- Credentials must be encrypted while at rest (database or file) with `attr_encrypted`. See [issue #26243](https://gitlab.com/gitlab-org/gitlab/-/issues/26243) before using `attr_encrypted`. + - Store the encryption keys separately from the encrypted credentials with proper access control. For instance, store the keys in a vault, KMS, or file. Here is an [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/user.rb#L70-74) use of `attr_encrypted` for encryption with keys stored in separate access controlled file. + - When the intention is to only compare secrets, store only the salted hash of the secret instead of the encrypted value. +- Never commit credentials to repositories. + - The [Gitleaks Git hook](https://gitlab.com/gitlab-com/gl-security/security-research/gitleaks-endpoint-installer) is recommended for preventing credentials from being committed. +- Never log credentials under any circumstance. Issue [#353857](https://gitlab.com/gitlab-org/gitlab/-/issues/353857) is an example of credential leaks through log file. +- When credentials are required in a CI/CD job, use [masked variables](../ci/variables/index.md#mask-a-cicd-variable) to help prevent accidental exposure in the job logs. Be aware that when [debug logging](../ci/variables/index.md#debug-logging) is enabled, all masked CI/CD variables are visible in job logs. Also consider using [protected variables](../ci/variables/index.md#protected-cicd-variables) when possible so that sensitive CI/CD variables are only available to pipelines running on protected branches or protected tags. +- Proper scanners must be enabled depending on what data those credentials are protecting. See the [Application Security Inventory Policy](https://about.gitlab.com/handbook/engineering/security/security-engineering-and-research/application-security/inventory.html#policies) and our [Data Classification Standards](https://about.gitlab.com/handbook/engineering/security/data-classification-standard.html#data-classification-standards). +- To store and/or share credentials between teams, refer to [1Password for Teams](https://about.gitlab.com/handbook/security/#1password-for-teams) and follow [the 1Password Guidelines](https://about.gitlab.com/handbook/security/#1password-guidelines). +- If you need to share a secret with a team member, use 1Password. Do not share a secret over email, Slack, or other service on the Internet. + +### In transit + +- Use an encrypted channel like TLS to transmit credentials. See [our TLS minimum recommendation guidelines](#tls-minimum-recommended-version). +- Avoid including credentials as part of an HTTP response unless it is absolutely necessary as part of the workflow. For example, generating a PAT for users. +- Avoid sending credentials in URL parameters, as these can be more easily logged inadvertently during transit. + +In the event of credential leak through an MR, issue, or any other medium, [reach out to SIRT team](https://about.gitlab.com/handbook/engineering/security/security-operations/sirt/#-engaging-sirt). diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md index ca4a0158051..27bc4d2e8ca 100644 --- a/doc/development/service_ping/implement.md +++ b/doc/development/service_ping/implement.md @@ -268,10 +268,9 @@ Arguments: #### Ordinary Redis counters -Examples of implementation: +Example of implementation: -- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) -- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) +Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) ##### `UsageData` API @@ -287,9 +286,7 @@ Enabled by default in GitLab 13.7 and later. Increment event count using an ordinary Redis counter, for a given event name. API requests are protected by checking for a valid CSRF token. - - To increment the values, the related feature `usage_data_<event_name>` must be enabled. - + ```plaintext POST /usage_data/increment_counter ``` @@ -366,7 +363,7 @@ Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PF aggregation. - `aggregation`: may be set to a `:daily` or `:weekly` key. Defines how counting data is stored in Redis. Aggregation on a `daily` basis does not pull more fine grained data. - - `feature_flag`: optional `default_enabled: :yaml`. If no feature flag is set then the tracking is enabled. One feature flag can be used for multiple events. For details, see our [GitLab internal Feature flags](../feature_flags/index.md) documentation. The feature flags are owned by the group adding the event tracking. + - `feature_flag`: if no feature flag is set then the tracking is enabled. One feature flag can be used for multiple events. For details, see our [GitLab internal Feature flags](../feature_flags/index.md) documentation. The feature flags are owned by the group adding the event tracking. 1. Use one of the following methods to track the event: @@ -580,7 +577,6 @@ Example: ```ruby # Redis Counters redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } # Define events in common.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml diff --git a/doc/development/service_ping/index.md b/doc/development/service_ping/index.md index 14bb90537e7..1e09dada36e 100644 --- a/doc/development/service_ping/index.md +++ b/doc/development/service_ping/index.md @@ -22,7 +22,7 @@ and sales teams understand how GitLab is used. The data helps to: Service Ping information is not anonymous. It's linked to the instance's hostname, but does not contain project names, usernames, or any other specific data. -Sending a Service Ping payload is optional and you can [disable](#disable-service-ping) it on any +Sending a Service Ping payload is optional and you can [disable](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics) it on any self-managed instance. When Service Ping is enabled, GitLab gathers data from the other instances and can show your instance's usage statistics to your users. @@ -38,23 +38,6 @@ We use the following terminology to describe the Service Ping components: - **MAU**: monthly active users. - **WAU**: weekly active users. -### Why enable Service Ping? - -The main purpose of Service Ping is to build a better GitLab. We collect data about how GitLab is used -to understand feature or stage adoption and usage. This data gives an insight into how GitLab adds -value and helps our team understand the reasons why people use GitLab, and with this knowledge we're able to -make better product decisions. - -There are several other benefits to enabling Service Ping: - -- As a benefit of having Service Ping active, GitLab lets you analyze the users' activities over time of your GitLab installation. -- As a benefit of having Service Ping active, GitLab provides you with [DevOps Score](../../user/admin_area/analytics/dev_ops_reports.md#devops-score), which gives you an overview of your entire instance's adoption of Concurrent DevOps from planning to monitoring. -- You get better, more proactive support (assuming that our TAMs and support organization used the data to deliver more value). -- You get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? -- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. -- Service Ping is enabled by default. To disable it, see [Disable Service Ping](#disable-service-ping). -- When Service Ping is enabled, you have the option to participate in our [Registration Features Program](#registration-features-program) and receive free paid features. - ### Limitations - Service Ping does not track frontend events things like page views, link clicks, or user sessions. @@ -65,107 +48,6 @@ Because of these limitations we recommend you: - Instrument your products with Snowplow for more detailed analytics on GitLab.com. - Use Service Ping to track aggregated backend events on self-managed instances. -### Registration Features Program - -> Introduced in GitLab 14.1. - -In GitLab versions 14.1 and later, GitLab Free customers with a self-managed instance running -[GitLab EE](../ee_features.md) can receive paid features by registering with GitLab and sending us -activity data through Service Ping. Features introduced here do not remove the feature from its paid -tier. Users can continue to access the features in a paid tier without sharing usage data. - -#### Features available in 14.1 and later - -1. [Email from GitLab](../../user/admin_area/email_from_gitlab.md). - -#### Features available in 14.4 and later - -1. [Repository size limit](../../user/admin_area/settings/account_and_limit_settings.md#repository-size-limit). -1. [Restrict group access by IP address](../../user/group/index.md#restrict-group-access-by-ip-address). - -NOTE: -Registration is not yet required for participation, but will be added in a future milestone. - -#### Enable Registration Features - -1. Sign in as a user with administrator access. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. If not enabled, select the **Enable Service Ping** checkbox. -1. Select the **Enable Registration Features** checkbox. -1. Select **Save changes**. - -## View the Service Ping payload **(FREE SELF)** - -You can view the exact JSON payload sent to GitLab Inc. in the Admin Area. To view the payload: - -1. Sign in as a user with administrator access. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Select **Preview payload**. - -For an example payload, see [Example Service Ping payload](#example-service-ping-payload). - -## Disable Service Ping **(FREE SELF)** - -NOTE: -The method to disable Service Ping in the GitLab configuration file does not work in -GitLab versions 9.3 to 13.12.3. See the [troubleshooting section](#cannot-disable-service-ping-using-the-configuration-file) -on how to disable it. - -You can disable Service Ping either using the GitLab UI, or editing the GitLab -configuration file. - -### Disable Service Ping using the UI - -To disable Service Ping in the GitLab UI: - -1. Sign in as a user with administrator access. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Clear the **Enable Service Ping** checkbox. -1. Select **Save changes**. - -### Disable Service Ping using the configuration file - -To disable Service Ping and prevent it from being configured in the future through -the Admin Area: - -**For installations using the Linux package:** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['usage_ping_enabled'] = false - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -**For installations from source:** - -1. Edit `/home/git/gitlab/config/gitlab.yml`: - - ```yaml - production: &base - # ... - gitlab: - # ... - usage_ping_enabled: false - ``` - -1. Restart GitLab: - - ```shell - sudo service gitlab restart - ``` - ## Service Ping request flow The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, the GitLab S3 Bucket, the GitLab Snowflake Data Warehouse, and Sisense: @@ -211,23 +93,53 @@ sequenceDiagram the required URL is <https://version.gitlab.com/>. 1. In case of an error, it will be reported to the Version application along with following pieces of information: -- `uuid` - GitLab instance unique identifier -- `hostname` - GitLab instance hostname -- `version` - GitLab instance current versions -- `elapsed` - Amount of time which passed since Service Ping report process started and moment of error occurrence -- `message` - Error message - -<pre> -<code> -{ - "uuid"=>"02333324-1cd7-4c3b-a45b-a4993f05fb1d", - "hostname"=>"127.0.0.1", - "version"=>"14.7.0-pre", - "elapsed"=>0.006946, - "message"=>'PG::UndefinedColumn: ERROR: column \"non_existent_attribute\" does not exist\nLINE 1: SELECT COUNT(non_existent_attribute) FROM \"issues\" /*applica...' -} -</code> -</pre> + - `uuid` - GitLab instance unique identifier + - `hostname` - GitLab instance hostname + - `version` - GitLab instance current versions + - `elapsed` - Amount of time which passed since Service Ping report process started and moment of error occurrence + - `message` - Error message + + <pre> + <code> + { + "uuid"=>"02333324-1cd7-4c3b-a45b-a4993f05fb1d", + "hostname"=>"127.0.0.1", + "version"=>"14.7.0-pre", + "elapsed"=>0.006946, + "message"=>'PG::UndefinedColumn: ERROR: column \"non_existent_attribute\" does not exist\nLINE 1: SELECT COUNT(non_existent_attribute) FROM \"issues\" /*applica...' + } + </code> + </pre> + +1. Finally, the timing metadata information that is used for diagnostic purposes is submitted to the Versions application. It consists of a list of metric identifiers and the time it took to calculate the metrics: + + > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37911) in GitLab 15.0 [with a flag(../../user/feature_flags.md), enabled by default. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `measure_service_ping_metric_collection`. +On GitLab.com, this feature is available. + +```ruby + {"metadata"=> + {"metrics"=> + [{"name"=>"version", "time_elapsed"=>1.1811964213848114e-05}, + {"name"=>"installation_type", "time_elapsed"=>0.00017242692410945892}, + {"name"=>"license_billable_users", "time_elapsed"=>0.009520471096038818}, + .... + {"name"=>"counts.clusters_platforms_eks", + "time_elapsed"=>0.05638605775311589}, + {"name"=>"counts.clusters_platforms_gke", + "time_elapsed"=>0.40995341585949063}, + {"name"=>"counts.clusters_platforms_user", + "time_elapsed"=>0.06410990096628666}, + {"name"=>"counts.clusters_management_project", + "time_elapsed"=>0.24020783510059118}, + {"name"=>"counts.clusters_integrations_elastic_stack", + "time_elapsed"=>0.03484998410567641} + ] + } + } + ``` ### On a Geo secondary site @@ -251,6 +163,25 @@ We also collect metrics specific to [Geo](../../administration/geo/index.md) sec ] ``` +### Enable or disable service ping metadata reporting + +Service Ping timing metadata reporting is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +Feature.enable(:measure_service_ping_metric_collection) +``` + +To disable it: + +```ruby +Feature.disable(:measure_service_ping_metric_collection) +``` + ## Implementing Service Ping See the [implement Service Ping](implement.md) guide. @@ -513,7 +444,7 @@ To generate Service Ping, use [Teleport](https://goteleport.com/docs/) or a deta ``` 1. Connect to console host: - + ```shell ssh $USER-rails@console-01-sv-gprd.c.gitlab-production.internal ``` @@ -526,11 +457,15 @@ To generate Service Ping, use [Teleport](https://goteleport.com/docs/) or a deta 1. To detach from screen, press `ctrl + A`, `ctrl + D`. 1. Exit from bastion: - + ```shell exit ``` +1. Get the metrics duration from logs: + +Search in Google Console logs for `time_elapsed`. Query example [here](https://cloudlogging.app.goo.gl/nWheZvD8D3nWazNe6). + ### Verification (After approx 30 hours) #### Verify with Teleport @@ -560,7 +495,7 @@ To generate Service Ping, use [Teleport](https://goteleport.com/docs/) or a deta ``` 1. Check the last payload in `raw_usage_data` table: - + ```shell RawUsageData.last.payload ``` @@ -580,115 +515,10 @@ skip_db_write: ServicePing::SubmitService.new(skip_db_write: true).execute ``` -## Manually upload Service Ping payload - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7388) in GitLab 14.8 with a flag named `admin_application_settings_service_usage_data_center`. Disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83265) in GitLab 14.10. - -Service Ping payload can be uploaded to GitLab even if your application instance doesn't have access to the internet, -or you don't have Service Ping [cron job](#how-service-ping-works) enabled. - -To upload payload manually: - -1. Sign in as a user with administrator access. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Service** usage data. -1. Select **Download payload**. -1. Save the JSON file. -1. Visit [Service usage data center](https://version.gitlab.com/usage_data/new). -1. Select **Choose file** and choose the file from p5. -1. Select **Upload**. - -The uploaded file is encrypted and sent using secure [HTTPS protocol](https://en.wikipedia.org/wiki/HTTPS). HTTPS creates a secure -communication channel between web browser and the server, and protects transmitted data against man-in-the-middle attacks. - ## Monitoring Service Ping reporting process state is monitored with [internal SiSense dashboard](https://app.periscopedata.com/app/gitlab/968489/Product-Intelligence---Service-Ping-Health). -## Troubleshooting - -### Cannot disable Service Ping using the configuration file - -The method to disable Service Ping using the GitLab configuration file does not work in -GitLab versions 9.3.0 to 13.12.3. To disable it, you must use the Admin Area in -the GitLab UI instead. For more information, see -[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/333269). - -GitLab functionality and application settings cannot override or circumvent -restrictions at the network layer. If Service Ping is blocked by your firewall, -you are not impacted by this bug. - -#### Check if you are affected - -You can check if you were affected by this bug by using the Admin Area or by -checking the configuration file of your GitLab instance: - -- Using the Admin Area: - - 1. On the top bar, select **Menu > Admin**. - 1. On the left sidebar, select **Settings > Metrics and profiling**. - 1. Expand **Usage Statistics**. - 1. Are you able to check or uncheck the checkbox to disable Service Ping? - - - If _yes_, your GitLab instance is not affected by this bug. - - If you can't check or uncheck the checkbox, you are affected by this bug. - See the steps on [how to fix this](#how-to-fix-the-cannot-disable-service-ping-bug). - -- Checking your GitLab instance configuration file: - - To check whether you're impacted by this bug, check your instance configuration - settings. The configuration file in which Service Ping can be disabled depends - on your installation and deployment method, but is typically one of the following: - - - `/etc/gitlab/gitlab.rb` for Omnibus GitLab Linux Package and Docker. - - `charts.yaml` for GitLab Helm and cloud-native Kubernetes deployments. - - `gitlab.yml` for GitLab installations from source. - - To check the relevant configuration file for strings that indicate whether - Service Ping is disabled, you can use `grep`: - - ```shell - # Linux package - grep "usage_ping_enabled'\] = false" /etc/gitlab/gitlab.rb - - # Kubernetes charts - grep "enableUsagePing: false" values.yaml - - # From source - grep "usage_ping_enabled'\] = false" gitlab/config.yml - ``` - - If you see any output after running the relevant command, your GitLab instance - may be affected by the bug. Otherwise, your instance is not affected. - -#### How to fix the "Cannot disable Service Ping" bug - -To work around this bug, you have two options: - -- [Update](../../update/index.md) to GitLab 13.12.4 or newer to fix this bug. -- If you can't update to GitLab 13.12.4 or newer, enable Service Ping in the - configuration file, then disable Service Ping in the UI. For example, if you're - using the Linux package: - - 1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['usage_ping_enabled'] = true - ``` - - 1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - - 1. In GitLab, on the top bar, select **Menu > Admin**. - 1. On the left sidebar, select **Settings > Metrics and profiling**. - 1. Expand **Usage Statistics**. - 1. Clear the **Enable Service Ping** checkbox. - 1. Select **Save Changes**. - ## Related topics - [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) diff --git a/doc/development/service_ping/metrics_dictionary.md b/doc/development/service_ping/metrics_dictionary.md index ab3d301908b..ead11a412fa 100644 --- a/doc/development/service_ping/metrics_dictionary.md +++ b/doc/development/service_ping/metrics_dictionary.md @@ -25,7 +25,7 @@ All metrics are stored in YAML files: - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) WARNING: -Only metrics with a metric definition YAML are added to the Service Ping JSON payload. +Only metrics with a metric definition YAML and whose status is not `removed` are added to the Service Ping JSON payload. Each metric is defined in a separate YAML file consisting of a number of fields: @@ -50,6 +50,7 @@ Each metric is defined in a separate YAML file consisting of a number of fields: | `milestone` | no | The milestone when the metric is introduced and when it's available to self-managed instances with the official GitLab release. | | `milestone_removed` | no | The milestone when the metric is removed. | | `introduced_by_url` | no | The URL to the merge request that introduced the metric to be available for self-managed instances. | +| `removed_by_url` | no | The URL to the merge request that removed the metric. | | `repair_issue_url` | no | The URL of the issue that was created to repair a metric with a `broken` status. | | `options` | no | `object`: options information needed to calculate the metric value. | | `skip_validation` | no | This should **not** be set. [Used for imported metrics until we review, update and make them valid](https://gitlab.com/groups/gitlab-org/-/epics/5425). | @@ -131,7 +132,7 @@ which has a related schema in `/config/metrics/objects_schemas/topology_schema.j We use the following categories to classify a metric: - `operational`: Required data for operational purposes. -- `optional`: Default value for a metric. Data that is optional to collect. This can be [enabled or disabled](../service_ping/index.md#disable-service-ping) in the Admin Area. +- `optional`: Default value for a metric. Data that is optional to collect. This can be [enabled or disabled](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics) in the Admin Area. - `subscription`: Data related to licensing. - `standard`: Standard set of identifiers that are included when collecting data. diff --git a/doc/development/service_ping/metrics_instrumentation.md b/doc/development/service_ping/metrics_instrumentation.md index 3d56f3e777f..e718d972fba 100644 --- a/doc/development/service_ping/metrics_instrumentation.md +++ b/doc/development/service_ping/metrics_instrumentation.md @@ -26,7 +26,7 @@ A metric definition has the [`instrumentation_class`](metrics_dictionary.md) fie The defined instrumentation class should inherit one of the existing metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric`, or `GenericMetric`. -The current convention is that a single instrumentation class corresponds to a single metric. On a rare occasions, there are exceptions to that convention like [Redis metrics](#redis-metrics). To use a single instrumentation class for more than one metric, please reach out to one of the `@gitlab-org/growth/product-intelligence/engineers` members to consult about your case. +The current convention is that a single instrumentation class corresponds to a single metric. On a rare occasions, there are exceptions to that convention like [Redis metrics](#redis-metrics). To use a single instrumentation class for more than one metric, please reach out to one of the `@gitlab-org/growth/product-intelligence/engineers` members to consult about your case. Using the instrumentation classes ensures that metrics can fail safe individually, without breaking the entire process of Service Ping generation. @@ -40,6 +40,7 @@ We have built a domain-specific language (DSL) to define the metrics instrumenta - `start`: Specifies the start value of the batch counting, by default is `relation.minimum(:id)`. - `finish`: Specifies the end value of the batch counting, by default is `relation.maximum(:id)`. - `cache_start_and_finish_as`: Specifies the cache key for `start` and `finish` values and sets up caching them. Use this call when `start` and `finish` are expensive queries that should be reused between different metric calculations. +- `available?`: Specifies whether the metric should be reported. The default is `true`. [Example of a merge request that adds a database metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60022). @@ -123,6 +124,37 @@ options: counter_class: SourceCodeCounter ``` +### Availability-restrained Redis metrics + +If the Redis metric should only be available in the report under some conditions, then you must specify these conditions in a new class that is a child of the `RedisMetric` class. + +```ruby +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class MergeUsageCountRedisMetric < RedisMetric + available? { Feature.enabled?(:merge_usage_data_missing_key_paths) } + end + end + end + end +end +``` + +You must also use the class's name in the YAML setup. + +```yaml +time_frame: all +data_source: redis +instrumentation_class: 'MergeUsageCountRedisMetric' +options: + event: pushes + counter_class: SourceCodeCounter +``` + ## Redis HyperLogLog metrics [Example of a merge request that adds a `RedisHLL` metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61685). @@ -138,8 +170,42 @@ options: - i_quickactions_approve ``` +### Availability-restrained Redis HyperLogLog metrics + +If the Redis HyperLogLog metric should only be available in the report under some conditions, then you must specify these conditions in a new class that is a child of the `RedisHLLMetric` class. + +```ruby +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class MergeUsageCountRedisHLLMetric < RedisHLLMetric + available? { Feature.enabled?(:merge_usage_data_missing_key_paths) } + end + end + end + end +end +``` + +You must also use the class's name in the YAML setup. + +```yaml +time_frame: 28d +data_source: redis_hll +instrumentation_class: 'MergeUsageCountRedisHLLMetric' +options: + events: + - i_quickactions_approve +``` + ## Generic metrics +- `value`: Specifies the value of the metric. +- `available?`: Specifies whether the metric should be reported. The default is `true`. + [Example of a merge request that adds a generic metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60256). ```ruby diff --git a/doc/development/service_ping/metrics_lifecycle.md b/doc/development/service_ping/metrics_lifecycle.md index 844c989c640..c9cc9a4f2d2 100644 --- a/doc/development/service_ping/metrics_lifecycle.md +++ b/doc/development/service_ping/metrics_lifecycle.md @@ -113,6 +113,7 @@ To remove a metric: update the attributes of the metric's YAML definition: - Set the `status:` to `removed`. + - Set `removed_by_url:` to the URL of the MR removing the metric - Set `milestone_removed:` to the number of the milestone in which the metric was removed. diff --git a/doc/development/service_ping/troubleshooting.md b/doc/development/service_ping/troubleshooting.md index 15bc01f1270..2764ef41f98 100644 --- a/doc/development/service_ping/troubleshooting.md +++ b/doc/development/service_ping/troubleshooting.md @@ -22,10 +22,91 @@ The alert compares the current daily value with the daily value from previous we You can use [this query](https://gitlab.com/gitlab-org/gitlab/-/issues/347298#note_836685350) as an example, to start detecting when the drop started. -### Troubleshooting GitLab application layer +### Troubleshoot the GitLab application layer For results about an investigation conducted into an unexpected drop in Service ping Payload events volume, see [this issue](https://gitlab.com/gitlab-data/analytics/-/issues/11071). -### Troubleshooting data warehouse layer +### Troubleshoot the data warehouse layer Reach out to the [Data team](https://about.gitlab.com/handbook/business-technology/data-team/) to ask about current state of data warehouse. On their handbook page there is a [section with contact details](https://about.gitlab.com/handbook/business-technology/data-team/#how-to-connect-with-us). + +### Cannot disable Service Ping with the configuration file + +The method to disable Service Ping with the GitLab configuration file does not work in +GitLab versions 9.3.0 to 13.12.3. To disable it, you must use the Admin Area in +the GitLab UI instead. For more information, see +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/333269). + +GitLab functionality and application settings cannot override or circumvent +restrictions at the network layer. If Service Ping is blocked by your firewall, +you are not impacted by this bug. + +#### Check if you are affected + +You can check if you were affected by this bug by using the Admin Area or by +checking the configuration file of your GitLab instance: + +- Using the Admin Area: + + 1. On the top bar, select **Menu > Admin**. + 1. On the left sidebar, select **Settings > Metrics and profiling**. + 1. Expand **Usage Statistics**. + 1. Are you able to check or uncheck the checkbox to disable Service Ping? + + - If _yes_, your GitLab instance is not affected by this bug. + - If you can't check or uncheck the checkbox, you are affected by this bug. + See the steps on [how to fix this](#how-to-fix-the-cannot-disable-service-ping-bug). + +- Checking your GitLab instance configuration file: + + To check whether you're impacted by this bug, check your instance configuration + settings. The configuration file in which Service Ping can be disabled depends + on your installation and deployment method, but is typically one of the following: + + - `/etc/gitlab/gitlab.rb` for Omnibus GitLab Linux Package and Docker. + - `charts.yaml` for GitLab Helm and cloud-native Kubernetes deployments. + - `gitlab.yml` for GitLab installations from source. + + To check the relevant configuration file for strings that indicate whether + Service Ping is disabled, you can use `grep`: + + ```shell + # Linux package + grep "usage_ping_enabled'\] = false" /etc/gitlab/gitlab.rb + + # Kubernetes charts + grep "enableUsagePing: false" values.yaml + + # From source + grep "usage_ping_enabled'\] = false" gitlab/config.yml + ``` + + If you see any output after running the relevant command, your GitLab instance + may be affected by the bug. Otherwise, your instance is not affected. + +#### How to fix the "Cannot disable Service Ping" bug + +To work around this bug, you have two options: + +- [Update](../../update/index.md) to GitLab 13.12.4 or newer to fix this bug. +- If you can't update to GitLab 13.12.4 or newer, enable Service Ping in the + configuration file, then disable Service Ping in the UI. For example, if you're + using the Linux package: + + 1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['usage_ping_enabled'] = true + ``` + + 1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + + 1. In GitLab, on the top bar, select **Menu > Admin**. + 1. On the left sidebar, select **Settings > Metrics and profiling**. + 1. Expand **Usage Statistics**. + 1. Clear the **Enable Service Ping** checkbox. + 1. Select **Save Changes**. diff --git a/doc/development/sidekiq/idempotent_jobs.md b/doc/development/sidekiq/idempotent_jobs.md index 38db22f8467..a5ae8737ad1 100644 --- a/doc/development/sidekiq/idempotent_jobs.md +++ b/doc/development/sidekiq/idempotent_jobs.md @@ -135,7 +135,7 @@ happened. See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/342123) GitLab doesn't skip jobs scheduled in the future, as we assume that the state has changed by the time the job is scheduled to -execute. Deduplication of jobs scheduled in the feature is possible +execute. Deduplication of jobs scheduled in the future is possible for both `until_executed` and `until_executing` strategies. If you do want to deduplicate jobs scheduled in the future, diff --git a/doc/development/sidekiq_style_guide.md b/doc/development/sidekiq_style_guide.md deleted file mode 100644 index 1b5e7addf29..00000000000 --- a/doc/development/sidekiq_style_guide.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'sidekiq/index.md' -remove_date: '2022-04-13' ---- - -This document was moved to [another location](sidekiq/index.md). - -<!-- This redirect file can be deleted after <2022-04-13>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/snowplow/implementation.md b/doc/development/snowplow/implementation.md index 162b77772f9..f4123e3ba86 100644 --- a/doc/development/snowplow/implementation.md +++ b/doc/development/snowplow/implementation.md @@ -36,7 +36,7 @@ as base: _\* Undergoes a pseudonymization process at the collector level._ -These properties [are overriden](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/tracking/get_standard_context.js) +These properties [are overridden](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/tracking/get_standard_context.js) with frontend-specific values, like `source` (`gitlab-javascript`), `google_analytics_id` and the custom `extra` object. You can modify this object for any subsequent structured event that fires, although this is not recommended. @@ -83,7 +83,7 @@ The following example shows `data-track-*` attributes assigned to a button: | `data-track-action` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field is `activate_form_input` and clicking a button is `click_button`. Replaces `data-track-event`, which was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/290962) in GitLab 13.11. | | `data-track-label` | false | The specific element or object to act on. This can be: the label of the element, for example, a tab labeled 'Create from template' for `create_from_template`; a unique identifier if no text is available, for example, `groups_dropdown_close` for closing the Groups dropdown in the top bar; or the name or title attribute of a record being created. | | `data-track-property` | false | Any additional property of the element, or object being acted on. | -| `data-track-value` | false | Describes a numeric value (decimal) directly related to the event. This could be the value of an input. For example, `10` when clicking `internal` visibility. If omitted, this is the element's `value` property or `undefined`. For checkboxes, the default value is the element's checked attribute or `0` when unchecked. The value is parsed as numeric before sendind the event. | +| `data-track-value` | false | Describes a numeric value (decimal) directly related to the event. This could be the value of an input. For example, `10` when clicking `internal` visibility. If omitted, this is the element's `value` property or `undefined`. For checkboxes, the default value is the element's checked attribute or `0` when unchecked. The value is parsed as numeric before sending the event. | | `data-track-extra` | false | A key-value pair object passed as a valid JSON string. This attribute is added to the `extra` property in our [`gitlab_standard`](schemas.md#gitlab_standard) schema. | | `data-track-context` | false | To append a custom context object, passed as a valid JSON string. | @@ -97,10 +97,12 @@ If click events stop propagating, you must implement listeners and [Vue componen #### Helper methods -You can use the following Ruby helper: +You can use the following Ruby helpers: ```ruby tracking_attrs(label, action, property) # { data: { track_label... } } + +tracking_attrs_data(label, action, property) # { track_label... } ``` You can also use it on HAML templates: @@ -108,8 +110,8 @@ You can also use it on HAML templates: ```haml %button{ **tracking_attrs('main_navigation', 'click_button', 'navigation') } -// When adding additional data -// %button{ data: { platform: "...", **tracking_attrs('main_navigation', 'click_button', 'navigation') } } +// When merging with additional data +// %button{ data: { platform: "...", **tracking_attrs_data('main_navigation', 'click_button', 'navigation') } } ``` If you use the GitLab helper method [`nav_link`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/helpers/tab_helper.rb#L76), you must wrap `html_options` under the `html_options` keyword argument. If you diff --git a/doc/development/snowplow/troubleshooting.md b/doc/development/snowplow/troubleshooting.md index 47d775d89aa..2a6db80a6f2 100644 --- a/doc/development/snowplow/troubleshooting.md +++ b/doc/development/snowplow/troubleshooting.md @@ -21,7 +21,7 @@ While on CloudWatch dashboard set time range to last 4 weeks, to get better pict 1. `ELB New Flow Count` and `Collector Auto Scaling Group Network In/Out` - they show in order: number of connections to collectors via load balancers and data volume (in bytes) processed by collectors. If there is drop visible there, it means less events were fired from the GitLab application. Proceed to [application layer guide](#troubleshooting-gitlab-application-layer) for more details 1. `Firehose Records to S3` - it shows how many event records were saved to S3 bucket, if there was drop on this chart but not on the charts from 1. it means that problem is located at AWS infrastructure layer, please refer to [AWS layer guide](#troubleshooting-aws-layer) -1. If drop wasn't visible on any of previous charts it means that probelm is at data warehouse layer, please refer to [data warehouse layer guide](#troubleshooting-data-warehouse-layer) +1. If drop wasn't visible on any of previous charts it means that problem is at data warehouse layer, please refer to [data warehouse layer guide](#troubleshooting-data-warehouse-layer) ### Troubleshooting GitLab application layer @@ -48,3 +48,27 @@ Already conducted investigations: ### Troubleshooting data warehouse layer Reach out to [Data team](https://about.gitlab.com/handbook/business-technology/data-team/) to ask about current state of data warehouse. On their handbook page there is a [section with contact details](https://about.gitlab.com/handbook/business-technology/data-team/#how-to-connect-with-us) + +## Delay in Snowplow Enrichers + +If there is an alert for **Snowplow Raw Good Stream Backing Up**, we receive an email notification. This sometimes happens because Snowplow Enrichers don't scale well enough for the amount of Snowplow events. + +If the delay goes over 48 hours, we lose data. + +### Contact SRE on-call + +Send a message in the [#infrastructure_lounge](https://gitlab.slack.com/archives/CB3LSMEJV) Slack channel using the following template: + +```markdown +Hello team! + +We received an alert for [Snowplow Raw Good Stream Backing Up](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#alarmsV2:alarm/SnowPlow+Raw+Good+Stream+Backing+Up?). + +Enrichers are not scalling well for the amount of events we receive. + +See the [dashboard](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards:name=SnowPlow). + +Could we get assistance in order to fix the delay? + +Thank you! +``` diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md index 27a87d25170..39a3e3445ea 100644 --- a/doc/development/testing_guide/end_to_end/beginners_guide.md +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -49,7 +49,7 @@ For information about the distribution of tests per level in GitLab, see - Review how often the feature changes. Stable features that don't change very often might not be worth covering with end-to-end tests if they are already covered in lower level tests. -- Finally, discuss the proposed test with the developer(s) involved in implementing +- Finally, discuss the proposed test with the developers involved in implementing the feature and the lower-level tests. WARNING: diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index 47ebef37a4d..b4ec9e8ccd3 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -23,7 +23,7 @@ Please be sure to include the `feature_flag` tag so that the test can be skipped `name` - Format: `feature_flag: { name: 'feature_flag_name' }` -- Used only for informational purposes at this time. It should be included to help quickly determine what +- Used only for informational purposes at this time. It should be included to help quickly determine what feature flag is under test. `scope` @@ -31,28 +31,28 @@ feature flag is under test. - Format: `feature_flag: { name: 'feature_flag_name', scope: :project }` - When `scope` is set to `:global`, the test will be **skipped on all live .com environments**. This is to avoid issues with feature flag changes affecting other tests or users on that environment. - When `scope` is set to any other value (such as `:project`, `:group` or `:user`), or if no `scope` is specified, the test will only be **skipped on canary and production**. -This is due to the fact that admin access is not available there. +This is due to the fact that administrator access is not available there. **WARNING:** You are strongly advised to first try and [enable feature flags only for a group, project, user](../../feature_flags/index.md#feature-actors), or [feature group](../../feature_flags/index.md#feature-groups). -- If a global feature flag must be used, it is strongly recommended to apply `scope: :global` to the `feature_flag` metadata. This is, however, left up to the SET's discretion to determine the level of risk. - - For example, a test uses a global feature flag that only affects a small area of the application and is also needed to check for critical issues on live environments. - In such a scenario, it would be riskier to skip running the test. For cases like this, `scope` can be left out of the metadata so that it can still run in live environments - with admin access, such as staging. +- If a global feature flag must be used, it is strongly recommended to apply `scope: :global` to the `feature_flag` metadata. This is, however, left up to the SET's discretion to determine the level of risk. + - For example, a test uses a global feature flag that only affects a small area of the application and is also needed to check for critical issues on live environments. + In such a scenario, it would be riskier to skip running the test. For cases like this, `scope` can be left out of the metadata so that it can still run in live environments + with administrator access, such as staging. -**Note on `requires_admin`:** This tag should still be applied if there are other actions within the test that require admin access that are unrelated to updating a +**Note on `requires_admin`:** This tag should still be applied if there are other actions within the test that require administrator access that are unrelated to updating a feature flag (ex: creating a user via the API). The code below would enable a feature flag named `:feature_flag_name` for the project created by the test: ```ruby -RSpec.describe "with feature flag enabled", feature_flag: { - name: 'feature_flag_name', - scope: :project +RSpec.describe "with feature flag enabled", feature_flag: { + name: 'feature_flag_name', + scope: :project } do - + let(:project) { Resource::Project.fabricate_via_api! } before do diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 1e7cba9d247..9730115fd9f 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -147,11 +147,11 @@ as well as these: | Variable | Description | |-|-| | `QA_SCENARIO` | The scenario to run (default `Test::Instance::Image`) | -| `QA_TESTS` | The test(s) to run (no default, which means run all the tests in the scenario). Use file paths as you would when running tests via RSpec, for example, `qa/specs/features/ee/browser_ui` would include all the `EE` UI tests. | +| `QA_TESTS` | The tests to run (no default, which means run all the tests in the scenario). Use file paths as you would when running tests via RSpec, for example, `qa/specs/features/ee/browser_ui` would include all the `EE` UI tests. | | `QA_RSPEC_TAGS` | The RSpec tags to add (no default) | -For now [manual jobs with custom variables don't use the same variable -when retried](https://gitlab.com/gitlab-org/gitlab/-/issues/31367), so if you want to run the same test(s) multiple times, +For now, [manual jobs with custom variables don't use the same variable +when retried](https://gitlab.com/gitlab-org/gitlab/-/issues/31367), so if you want to run the same tests multiple times, specify the same variables in each `custom-parallel` job (up to as many of the 10 available jobs that you want to run). diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index 45161404c73..0163f2e648c 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -21,8 +21,8 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:github` | The test requires a GitHub personal access token. | | `:group_saml` | The test requires a GitLab instance that has SAML SSO enabled at the group level. Interacts with an external SAML identity provider. Paired with the `:orchestrated` tag. | | `:instance_saml` | The test requires a GitLab instance that has SAML SSO enabled at the instance level. Interacts with an external SAML identity provider. Paired with the `:orchestrated` tag. | -| `:integrations` | This aims to test the available [integrations](../../../user/project/integrations/overview.md#integrations-listing). The test requires Docker to be installed in the run context. It will provision the containers and can be run against a local instance or using the `gitlab-qa` scenario `Test::Integration::Integrations` | -| `:service_ping_disabled` | The test interacts with the GitLab configuration service ping at the instance level to turn admin setting service ping checkbox on or off. This tag will have the test run only in the `service_ping_disabled` job and must be paired with the `:orchestrated` and `:requires_admin` tags. | +| `:integrations` | This aims to test the available [integrations](../../../user/project/integrations/index.md#available-integrations). The test requires Docker to be installed in the run context. It will provision the containers and can be run against a local instance or using the `gitlab-qa` scenario `Test::Integration::Integrations` | +| `:service_ping_disabled` | The test interacts with the GitLab configuration service ping at the instance level to turn Admin Area setting service ping checkbox on or off. This tag will have the test run only in the `service_ping_disabled` job and must be paired with the `:orchestrated` and `:requires_admin` tags. | | `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) provisions the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. | | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test also includes provisioning of at least one Kubernetes cluster to test against. _This tag is often be paired with `:orchestrated`._ | | `:ldap_no_server` | The test requires a GitLab instance to be configured to use LDAP. To be used with the `:orchestrated` tag. It does not spin up an LDAP server at orchestration time. Instead, it creates the LDAP server at runtime. | diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index d03a4976a8c..d91c53823e2 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -297,7 +297,7 @@ it('tests a promise rejection', async () => { You can also simply return a promise from the test function. Using the `done` and `done.fail` callbacks is discouraged when working with -promises. They should only be used when testing callback-based code. +promises. They should not be used. **Bad**: @@ -466,18 +466,22 @@ it('waits for an Ajax call', () => { #### Vue rendering -To wait until a Vue component is re-rendered, use either of the equivalent -[`Vue.nextTick()`](https://vuejs.org/v2/api/#Vue-nextTick) or `vm.$nextTick()`. +Use [`nextTick()`](https://vuejs.org/v2/api/#Vue-nextTick) to wait until a Vue component is +re-rendered. **in Jest:** ```javascript -it('renders something', () => { +import { nextTick } from 'vue'; + +// ... + +it('renders something', async () => { wrapper.setProps({ value: 'new value' }); - return wrapper.vm.$nextTick().then(() => { - expect(wrapper.text()).toBe('new value'); - }); + await nextTick(); + + expect(wrapper.text()).toBe('new value'); }); ``` @@ -487,15 +491,17 @@ If the application triggers an event that you need to wait for in your test, reg the assertions: ```javascript -it('waits for an event', done => { +it('waits for an event', () => { eventHub.$once('someEvent', eventHandler); someFunction(); - function eventHandler() { - expect(something).toBe('done'); - done(); - } + return new Promise((resolve) => { + function expectEventHandler() { + expect(something).toBe('done'); + resolve(); + } + }); }); ``` @@ -807,11 +813,14 @@ The following are examples of tests that work for Jest: ```javascript it('uses some HTML element', () => { - loadFixtures('some/page.html'); // loads spec/frontend/fixtures/some/page.html and adds it to the DOM + loadHTMLFixture('some/page.html'); // loads spec/frontend/fixtures/some/page.html and adds it to the DOM const element = document.getElementById('#my-id'); // ... + + // Jest does not clean up the DOM automatically + resetHTMLFixture(); }); ``` diff --git a/doc/development/testing_guide/img/k9s.png b/doc/development/testing_guide/img/k9s.png Binary files differdeleted file mode 100644 index 34585b2a43a..00000000000 --- a/doc/development/testing_guide/img/k9s.png +++ /dev/null diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index f5483a4b79c..ff4b77dec2c 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -16,6 +16,7 @@ For any of the following scenarios, the `start-review-app-pipeline` job would be - for merge requests with frontend changes - for merge requests with changes to `{,ee/,jh/}{app/controllers}/**/*` - for merge requests with changes to `{,ee/,jh/}{app/models}/**/*` +- for merge requests with changes to `{,ee/,jh/}lib/{,ee/,jh/}gitlab/**/*` - for merge requests with QA changes - for scheduled pipelines - the MR has the `pipeline:run-review-app` label set @@ -198,7 +199,7 @@ subgraph "CNG-mirror pipeline" issue with a link to your merge request. Note that the deployment failure can reveal an actual problem introduced in your merge request (that is, this isn't necessarily a transient failure)! -- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing (note that we already retry them once), +- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing, please check the job's logs: you could discover an actual problem introduced in your merge request. You can also download the artifacts to see screenshots of the page at the time the failures occurred. If you don't find the cause of the diff --git a/doc/development/testing_guide/testing_migrations_guide.md b/doc/development/testing_guide/testing_migrations_guide.md index 4092c1a2f6d..d71788e21f3 100644 --- a/doc/development/testing_guide/testing_migrations_guide.md +++ b/doc/development/testing_guide/testing_migrations_guide.md @@ -227,6 +227,18 @@ expect('MigrationClass').to have_scheduled_batched_migration( ) ``` +#### `be_finalize_background_migration_of` + +Verifies that a migration calls `finalize_background_migration` with the expected background migration class. + +```ruby +# Migration +finalize_background_migration('MigrationClass') + +# Spec +expect(described_class).to be_finalize_background_migration_of('MigrationClass') +``` + ### Examples of migration tests Migration tests depend on what the migration does exactly, the most common types are data migrations and scheduling background migrations. diff --git a/doc/development/uploads/background.md b/doc/development/uploads/background.md index e68e4127b57..1ad1aec23f2 100644 --- a/doc/development/uploads/background.md +++ b/doc/development/uploads/background.md @@ -1,154 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-07-25' --- -# Uploads guide: Why GitLab uses custom upload logic +This document was moved to [another location](index.md). -This page is for developers trying to better understand the history behind GitLab uploads and the -technical challenges associated with uploads. - -## Problem description - -GitLab and [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) use special rules for handling file uploads, -because in an ordinary Rails application file uploads can become expensive as files grow in size. -Rails often sacrifices performance to provide a better developer experience, including how it handles -`multipart/form-post` uploads. In any Rack server, Rails applications included, when such a request arrives at the application server, -several things happen: - -1. A [Rack middleware](https://github.com/rack/rack/blob/main/lib/rack/multipart.rb) intercepts the request and parses the request body. -1. The middleware writes each file in the multipart request to a temporary directory on disk. -1. A `params` hash is constructed with entries pointing to the respective files on disk. -1. A Rails controller acts on the file contents. - -While this is convenient for developers, it is costly for the Ruby server process to buffer large files on disk. -Because of Ruby's [global interpreter lock](https://en.wikipedia.org/wiki/Global_interpreter_lock), -only a single thread of execution of a given Ruby process can be on CPU. This means the amount of CPU -time spent doing this is not available to other worker threads serving user requests. -Buffering files to disk also means spending more time in I/O routines and mode switches, which are expensive operations. - -The following diagram shows how GitLab handled such a request prior to putting optimizations in place. - -```mermaid -graph TB - subgraph "load balancers" - LB(Proxy) - end - - subgraph "Shared storage" - nfs(NFS) - end - - subgraph "redis cluster" - r(persisted redis) - end - LB-- 1 -->Workhorse - - subgraph "web or API fleet" - Workhorse-- 2 -->rails - end - rails-- "3 (write files)" -->nfs - rails-- "4 (schedule a job)" -->r - - subgraph sidekiq - s(sidekiq) - end - s-- "5 (fetch a job)" -->r - s-- "6 (read files)" -->nfs -``` - -We went through two major iterations of our uploads architecture to improve on these problems: - -1. [Moving disk buffering to Workhorse.](#moving-disk-buffering-to-workhorse) -1. [Uploading to Object Storage from Workhorse.](#moving-to-object-storage-and-direct-uploads) - -### Moving disk buffering to Workhorse - -To address the performance issues resulting from buffering files in Ruby, we moved this logic to Workhorse instead, -our reverse proxy fronting the GitLab Rails application. -Workhorse is written in Go, and is much better at dealing with stream processing and I/O than Rails. - -There are two parts to this implementation: - -1. In Workhorse, a request handler detects `multipart/form-data` content in an incoming user request. - If such a request is detected, Workhorse hijacks the request body before forwarding it to Rails. - Workhorse writes all files to disk, rewrites the multipart form fields to point to the new locations, signs the - request, then forwards it to Rails. -1. In Rails, a [custom multipart Rack middleware](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) - identifies any signed multipart requests coming from Workhorse and prepares the `params` hash Rails - would expect, now pointing to the files cached by Workhorse. This makes it a drop-in replacement for `Rack::Multipart`. - -The diagram below shows how GitLab handles such a request today: - -```mermaid -graph TB - subgraph "load balancers" - LB(HA Proxy) - end - - subgraph "Shared storage" - nfs(NFS) - end - - subgraph "redis cluster" - r(persisted redis) - end - LB-- 1 -->Workhorse - - subgraph "web or API fleet" - Workhorse-- "3 (without files)" -->rails - end - Workhorse -- "2 (write files)" -->nfs - rails-- "4 (schedule a job)" -->r - - subgraph sidekiq - s(sidekiq) - end - s-- "5 (fetch a job)" -->r - s-- "6 (read files)" -->nfs -``` - -While this "one-size-fits-all" solution greatly improves performance for multipart uploads without compromising -developer ergonomics, it severely limits GitLab [availability](#availability-challenges) -and [scalability](#scalability-challenges). - -#### Availability challenges - -Moving file buffering to Workhorse addresses the immediate performance problems stemming from Ruby not being good at -handling large file uploads. However, a remaining issue of this solution is its reliance on attached storage, -whether via ordinary hard drives or network attached storage like NFS. -NFS is a [single point of failure](https://en.wikipedia.org/wiki/Single_point_of_failure), and is unsuitable for -deploying GitLab in highly available, cloud native environments. - -#### Scalability challenges - -NFS is not a part of cloud native installations, such as those running in Kubernetes. -In Kubernetes, machine boundaries translate to pods, and without network-attached storage, disk-buffered uploads -must be written directly to the pod's file system. - -Using disk buffering presents us with a scalability challenge here. If Workhorse can only -write files to a pod's private file system, then these files are inaccessible outside of this particular pod. -With disk buffering, a Rails controller will accept a file upload and enqueue it for upload in a Sidekiq -background job. Therefore, Sidekiq requires access to these files. -However, in a cloud native environment all Sidekiq instances run on separate pods, so they are -not able to access files buffered to disk on a web server pod. - -Therefore, all features that involve Sidekiq uploading disk-buffered files severely limit the scalability of GitLab. - -## Moving to object storage and direct uploads - -To address these availability and scalability problems, -instead of buffering files to disk, we have added support for uploading files directly -from Workhorse to a given destination. While it remains possible to upload to local or network-attached storage -this way, you should use a highly available -[object store](https://en.wikipedia.org/wiki/Object_storage), -such as AWS S3, Google GCS, or Azure, for scalability reasons. - -With direct uploads, Workhorse does not buffer files to disk. Instead, it first authorizes the request with -the Rails application to find out where to upload it, then streams the file directly to its ultimate destination. - -To learn more about how disk buffering and direct uploads are implemented, see: - -- [How uploads work technically](implementation.md) -- [Adding new uploads](working_with_uploads.md) +<!-- This redirect file can be deleted after <2022-07-25>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/uploads/implementation.md b/doc/development/uploads/implementation.md index 13a875cd1af..1ad1aec23f2 100644 --- a/doc/development/uploads/implementation.md +++ b/doc/development/uploads/implementation.md @@ -1,190 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-07-25' --- -# Uploads guide: How uploads work technically +This document was moved to [another location](index.md). -This page is for developers trying to better understand what kinds of uploads exist in GitLab and how they are implemented. - -## Kinds of uploads and how to choose between them - -We can identify three major use-cases for an upload: - -1. **storage:** if we are uploading for storing a file (like artifacts, packages, or discussion attachments). In this case [direct upload](#direct-upload) is the proper level as it's the less resource-intensive operation. Additional information can be found on [File Storage in GitLab](../file_storage.md). -1. **in-controller/synchronous processing:** if we allow processing **small files** synchronously, using [disk buffered upload](#disk-buffered-upload) may speed up development. -1. **Sidekiq/asynchronous processing:** Asynchronous processing must implement [direct upload](#direct-upload), the reason being that it's the only way to support Cloud Native deployments without a shared NFS. - -Selecting the proper acceleration is a tradeoff between speed of development and operational costs. - -For more details about currently broken feature see [epic &1802](https://gitlab.com/groups/gitlab-org/-/epics/1802). - -### Handling repository uploads - -Some features involves Git repository uploads without using a regular Git client. -Some examples are uploading a repository file from the web interface and [design management](../../user/project/issues/design_management.md). - -Those uploads requires the rails controller to act as a Git client in lieu of the user. -Those operation falls into _in-controller/synchronous processing_ category, but we have no warranties on the file size. - -In case of a LFS upload, the file pointer is committed synchronously, but file upload to object storage is performed asynchronously with Sidekiq. - -## Upload encodings - -By upload encoding we mean how the file is included within the incoming request. - -We have three kinds of file encoding in our uploads: - -1. <i class="fa fa-check-circle"></i> **multipart**: `multipart/form-data` is the most common, a file is encoded as a part of a multipart encoded request. -1. <i class="fa fa-check-circle"></i> **body**: some APIs uploads files as the whole request body. -1. <i class="fa fa-times-circle"></i> **JSON**: some JSON APIs upload files as base64-encoded strings. This requires a change to GitLab Workhorse, - which is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/325068). - -## Uploading technologies - -By uploading technologies we mean how all the involved services interact with each other. - -GitLab supports 3 kinds of uploading technologies, here follows a brief description with a sequence diagram for each one. Diagrams are not meant to be exhaustive. - -### Rack Multipart upload - -This is the default kind of upload, and it's the most expensive in terms of resources. - -In this case, Workhorse is unaware of files being uploaded and acts as a regular proxy. - -When a multipart request reaches the rails application, `Rack::Multipart` leaves behind temporary files in `/tmp` and uses valuable Ruby process time to copy files around. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - - activate c - c ->>+w: POST /some/url/upload - w->>+r: POST /some/url/upload - - r->>r: save the incoming file on /tmp - r->>r: read the file for processing - - r-->>-c: request result - deactivate c - deactivate w -``` - -### Disk buffered upload - -This kind of upload avoids wasting resources caused by handling upload writes to `/tmp` in rails. - -This optimization is not active by default on REST API requests. - -When enabled, Workhorse looks for files in multipart MIME requests, uploading -any it finds to a temporary file on shared storage. The MIME data in the request -is replaced with the path to the corresponding file before it is forwarded to -Rails. - -To prevent abuse of this feature, Workhorse signs the modified request with a -special header, stating which entries it modified. Rails ignores any -unsigned path entries. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - participant s as NFS - - activate c - c ->>+w: POST /some/url/upload - - w->>+s: save the incoming file on a temporary location - s-->>-w: request result - - w->>+r: POST /some/url/upload - Note over w,r: file was replaced with its location<br>and other metadata - - opt requires async processing - r->>+redis: schedule a job - redis-->>-r: job is scheduled - end - - r-->>-c: request result - deactivate c - w->>-w: cleanup - - opt requires async processing - activate sidekiq - sidekiq->>+redis: fetch a job - redis-->>-sidekiq: job - - sidekiq->>+s: read file - s-->>-sidekiq: file - - sidekiq->>sidekiq: process file - - deactivate sidekiq - end -``` - -### Direct upload - -This is the more advanced acceleration technique we have in place. - -Workhorse asks Rails for temporary pre-signed object storage URLs and directly uploads to object storage. - -In this setup, an extra Rails route must be implemented in order to handle authorization. Examples of this can be found in: - -- [`Projects::LfsStorageController`](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/app/controllers/projects/lfs_storage_controller.rb) - and [its routes](https://gitlab.com/gitlab-org/gitlab/-/blob/cc723071ad337573e0360a879cbf99bc4fb7adb9/config/routes/git_http.rb#L31-32). -- [API endpoints for uploading packages](../packages.md#file-uploads). - -Direct upload falls back to _disk buffered upload_ when `direct_upload` is disabled inside the [object storage setting](../../administration/uploads.md#object-storage-settings). -The answer to the `/authorize` call contains only a file system path. - -```mermaid -sequenceDiagram - participant c as Client - participant w as Workhorse - participant r as Rails - participant os as Object Storage - - activate c - c ->>+w: POST /some/url/upload - - w ->>+r: POST /some/url/upload/authorize - Note over w,r: this request has an empty body - r-->>-w: presigned OS URL - - w->>+os: PUT file - Note over w,os: file is stored on a temporary location. Rails select the destination - os-->>-w: request result - - w->>+r: POST /some/url/upload - Note over w,r: file was replaced with its location<br>and other metadata - - r->>+os: move object to final destination - os-->>-r: request result - - opt requires async processing - r->>+redis: schedule a job - redis-->>-r: job is scheduled - end - - r-->>-c: request result - deactivate c - w->>-w: cleanup - - opt requires async processing - activate sidekiq - sidekiq->>+redis: fetch a job - redis-->>-sidekiq: job - - sidekiq->>+os: get object - os-->>-sidekiq: file - - sidekiq->>sidekiq: process file - - deactivate sidekiq - end -``` +<!-- This redirect file can be deleted after <2022-07-25>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/uploads/index.md b/doc/development/uploads/index.md index c486f2d3689..b8326489d40 100644 --- a/doc/development/uploads/index.md +++ b/doc/development/uploads/index.md @@ -6,9 +6,159 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Uploads development guide -Uploads are an integral part of many GitLab features. To understand how GitLab handles uploads, refer to -the following pages: +Uploads are an integral part of many GitLab features. To understand how GitLab handles uploads, this page +provides an overview of the key mechanisms for transferring files to a storage destination. -- [Why GitLab uses custom upload logic.](background.md) -- [How uploads work technically.](implementation.md) -- [How to add new uploads.](working_with_uploads.md) +GitLab uploads are configured by feature. All features that involve uploads provide the same configuration options, +but they can be configured independently of one another. For example, Git LFS uploads can be configured +independently of CI/CD build artifact uploads, but they both offer the same set of settings keys. These settings +govern how an upload is processed, which can have a dramatic impact on performance and scalability. + +This page summarizes the upload settings that are important in deciding how such files are handled. The sections +that follow then describe each of these mechanisms in more detail. + +## How upload settings drive upload flow + +Before we examine individual upload strategies in more detail, let's examine a high-level +breakdown of which upload settings map to each of these strategies. + +Upload settings themselves are documented in [Uploads administration](../../administration/uploads.md). +Here, we focus on how these settings drive the internals of GitLab upload logic. +At the top level, we distinguish between two **destinations** for uploaded files: + +- [**Local storage**](#local-storage) - Files are stored on a volume attached to the web server node. +- [**Object storage**](#object-storage) - Files are stored in a remote object store bucket. + +In this table, `x.y.z` specifies the path taken through `gitlab.yml`: + +| Setting | Value | Behavior | +| -------------------------------------- | ------- | ------------------------------- | +| `<feature>.object_store.enabled` | `false` | Files are stored locally in `<feature>.storage_path` | +| `<feature>.object_store.enabled` | `true` | Files are stored remotely in `<feature>.object_store.remote_directory` | + +When using object storage, administrators can control how those files are moved into the respective bucket. +This move can happen in one of these ways: + +- [Rails controller upload](#rails-controller-upload). +- [Background upload](#background-upload). +- [Direct upload](#direct-upload). + +These strategies activate as per the following `<feature>.object_store.*` settings: + +| | `background_upload` = `false` | `background_upload` = `true` | +| ------------------------- | ----------------------------- | ------------------------------- | +| `direct_upload` = `false` | Controller upload | Background upload | +| `direct_upload` = `true` | Direct upload | Direct upload (takes precedence)| + +Individual Sidekiq workers might also store files in object storage, which is not something we cover here. +More importantly, `background_upload` does not imply _all files are uploaded by Sidekiq._ +Sidekiq workers that store files in object storage could still exist when this setting is `false`. +Those cases are never user-initiated uploads, but they might occur in response to another user-initiated +action, such as exporting a GitLab repository. + +Finally, Workhorse assists most user-initiated uploads using an upload buffering mechanism to keep slow work out of Rails controllers. +This mechanism is explained in [Workhorse assisted uploads](#workhorse-assisted-uploads), +as it runs orthogonal to much of what we discuss beforehand. + +We now look at each case in more detail. + +## Local storage + +Local storage is the simplest path an upload can take. It was how GitLab treated uploads in its early days. +It assumes a storage volume (like a disk or network attached storage) is accessible +to the Rails application at `storage_path`. This file path is relative to the Rails root directory and, +like any upload setting, configurable per feature. + +When a client sends a file upload, Workhorse first buffers the file to disk, a mechanism explained in more +detail in [Workhorse assisted uploads](#workhorse-assisted-uploads). When the request reaches the Rails +application, the file already exists on local storage, so Rails merely has to move it to the specified +directory to finalize the transaction. + +Local storage cannot be used with cloud-native GitLab (CNG) installations. It is therefore not used for +GitLab SaaS either. + +## Object storage + +To provide horizontally scalable storage, you must use an object store provider such as: + +- Amazon AWS. +- Google Cloud Storage (GCS). +- Azure Cloud Storage. + +Using object storage provides two main benefits: + +- Ease of adding more storage capacity: cloud providers do this for you automatically. +- Enabling horizontal scaling of your GitLab installation: multiple GitLab application servers can access the same data + when it is stored in object storage. + +CNG installations including GitLab SaaS always use object storage (GCS in the case of GitLab SaaS.) + +A challenge with uploading to a remote object store is that it includes an outgoing HTTP request from +GitLab to the object store provider. As mentioned above, there are three different strategies available for how +this HTTP request is sent. + +- [Rails controller upload](#rails-controller-upload). +- [Background upload](#background-upload). +- [Direct upload](#direct-upload). + +### Rails controller upload + +When neither background upload nor direct upload are available, Rails uploads the file to object storage +as part of the controller `create` action. Which controller is responsible depends on the kind of file uploaded. + +A Rails controller upload is very similar to uploading to local storage. The main difference: Rails must +send an HTTP request to the object store. This happens via the [CarrierWave Fog](https://github.com/carrierwaveuploader/carrierwave#fog) +uploader. + +As with local storage, this strategy benefits from [Workhorse assistance](#workhorse-assisted-uploads) to +keep some of the costly I/O work out of Ruby and Rails. Direct upload does a better job at this because it also keeps the HTTP PUT requests to object storage outside Puma. + +This strategy is only suitable for small file uploads, as it is subject to Puma's 60 second request timeout. + +### Background upload + +WARNING: +This strategy is deprecated in GitLab 14.9 and later, and is scheduled to [be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/26600). + +With background uploads enabled: + +1. Files are uploaded as if they were to reside in local storage. +1. When Rails saves the upload metadata and the transaction completes, a Sidekiq job is scheduled. +1. The Sidekiq job transfers the file to the object store bucket. + - If the job completes, the upload record is updated to reflect the file's new location. + - If the job fails or gets lost, the upload stays in local storage and has the lifecycle of a normal local storage upload. + +As Rails and Sidekiq must cooperate to move the file to its final destination, it requires shared +storage and as such is unsuitable for CNG installations. We do not use background upload in GitLab SaaS. + +As background upload is an extension of local storage, it benefits from the same [Workhorse assistance](#workhorse-assisted-uploads) to +keep costly I/O work out of Ruby and Rails. + +### Direct upload + +Direct upload is the recommended way to move large files into object storage in CNG installations like GitLab SaaS. + +With direct upload enabled, Workhorse: + +1. Authorizes the request with Rails. +1. Establishes a connection with the object store itself to transfer the file to a temporary location. +1. When the transfer is complete, Workhorse finalizes the request with Rails. Rails issues an object store copy operation to put the file in its final location. +1. Completes the upload by deleting the temporary file in object storage. + +This strategy is a different form of [Workhorse assistance](#workhorse-assisted-uploads). It does not rely on shared storage that is accessible by both Workhorse and Puma. + +Of all existing upload strategies, direct upload is best able to handle large (gigabyte) uploads. However, because Puma still does an object storage copy operation, which takes time proportional to the size of the upload, there remains a possibility of hitting Puma timeouts. + +## Workhorse assisted uploads + +Most uploads receive assistance from Workhorse in some way. + +- Often, Workhorse buffers the upload to a temporary file. Workhorse adds metadata to the request to tell + Puma the name and location of the temporary file. This requires shared temporary storage between Workhorse and Puma. + All GitLab installations (including CNG) have this shared temporary storage. +- Workhorse sometimes pre-processes the file. For example, for CI artifact uploads, Workhorse creates a separate index + of the contents of the ZIP file. By doing this in Workhorse we bypass the Puma request timeout. + Compared to Sidekiq background processing, this has the advantage that the user does not see an intermediate state + where GitLab accepts the file but has not yet processed it. +- With direct upload, Workhorse can both pre-process the file and upload it to object storage. + Uploading a large file to object storage takes time; by doing this in Workhorse we avoid the Puma request timeout. diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md index 99c04888804..4e907530a9f 100644 --- a/doc/development/uploads/working_with_uploads.md +++ b/doc/development/uploads/working_with_uploads.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Uploads guide: Adding new uploads -In this section, we describe how to add a new upload route [accelerated](implementation.md#uploading-technologies) by Workhorse for [body and multipart](implementation.md#upload-encodings) encoded uploads. +Here, we describe how to add a new upload route [accelerated](index.md#workhorse-assisted-uploads) by Workhorse. Upload routes belong to one of these categories: @@ -15,31 +15,31 @@ Upload routes belong to one of these categories: 1. GraphQL API: uploads handled by a GraphQL resolve function. WARNING: -GraphQL uploads do not support [direct upload](implementation.md#direct-upload) yet. Depending on the use case, the feature may not work on installations without NFS (like GitLab.com or Kubernetes installations). Uploading to object storage inside the GraphQL resolve function may result in timeout errors. For more details please follow [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). +GraphQL uploads do not support [direct upload](index.md#direct-upload). Depending on the use case, the feature may not work on installations without NFS (like GitLab.com or Kubernetes installations). Uploading to object storage inside the GraphQL resolve function may result in timeout errors. For more details, follow [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). ## Update Workhorse for the new route -For both the Rails controller and Grape API uploads, Workhorse has to be updated in order to get the +For both the Rails controller and Grape API uploads, Workhorse must be updated to get the support for the new upload route. 1. Open a new issue in the [Workhorse tracker](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/new) describing precisely the new upload route: - The route's URL. - - The [upload encoding](implementation.md#upload-encodings). + - The upload encoding. - If possible, provide a dump of the upload request. 1. Implement and get the MR merged for this issue above. -1. Ask the Maintainers of [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to create a new release. You can do that in the MR - directly during the maintainer review or ask for it in the `#workhorse` Slack channel. +1. Ask the Maintainers of [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to create a new release. You can do that in the merge request + directly during the maintainer review, or ask for it in the `#workhorse` Slack channel. 1. Bump the [Workhorse version file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) to the version you have from the previous points, or bump it in the same merge request that contains - the Rails changes (see [Implementing the new route with a Rails controller](#implementing-the-new-route-with-a-rails-controller) or [Implementing the new route with a Grape API endpoint](#implementing-the-new-route-with-a-grape-api-endpoint) below). + the Rails changes. Refer to [Implementing the new route with a Rails controller](#implementing-the-new-route-with-a-rails-controller) or [Implementing the new route with a Grape API endpoint](#implementing-the-new-route-with-a-grape-api-endpoint) below. ## Implementing the new route with a Rails controller -For a Rails controller upload, we usually have a [multipart](implementation.md#upload-encodings) upload and there are a +For a Rails controller upload, we usually have a `multipart/form-data` upload and there are a few things to do: 1. The upload is available under the parameter name you're using. For example, it could be an `artifact` - or a nested parameter such as `user[avatar]`. Let's say that we have the upload under the + or a nested parameter such as `user[avatar]`. If you have the upload under the `file` parameter, reading `params[:file]` should get you an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance. 1. Generally speaking, it's a good idea to check if the instance is from the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) class. For example, see how we checked [that the parameter is indeed an `UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/commit/ea30fe8a71bf16ba07f1050ab4820607b5658719#51c0cc7a17b7f12c32bc41cfab3649ff2739b0eb_79_77). @@ -53,7 +53,7 @@ builds automatically for you. ## Implementing the new route with a Grape API endpoint -For a Grape API upload, we can have [body or a multipart](implementation.md#upload-encodings) upload. Things are slightly more complicated: two endpoints are needed. One for the +For a Grape API upload, we can have a body or multipart upload. Things are slightly more complicated: two endpoints are needed. One for the Workhorse pre-upload authorization and one for accepting the upload metadata from Workhorse: 1. Implement an endpoint with the URL + `/authorize` suffix that will: @@ -70,8 +70,8 @@ use `requires :file, type: ::API::Validations::Types::WorkhorseFile`. - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the [API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). - Check the user permissions. - - The remaining code of the processing. This is where the code must be reading the parameter (for -our example, it would be `params[:file]`). + - The remaining code of the processing. In this step, the code must read the parameter. For +our example, it would be `params[:file]`. WARNING: **Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) @@ -124,40 +124,40 @@ Therefore, document new uploads here by slotting them into the following tables: ### CarrierWave integration -| File | Carrierwave usage | Categorized | +| File | CarrierWave usage | Categorized | |---------------------------------------------------------|----------------------------------------------------------------------------------|---------------------| -| `app/models/project.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/projects/topic.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/group.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/user.rb` | `include Avatarable` | :white_check_mark: | -| `app/models/terraform/state_version.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/ci/job_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/ci/pipeline_artifact.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/pages_deployment.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/lfs_object.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/dependency_proxy/blob.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/dependency_proxy/manifest.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/packages/composer/cache_file.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/packages/package_file.rb` | `include FileStoreMounter` | :white_check_mark: | -| `app/models/concerns/packages/debian/component_file.rb` | `include FileStoreMounter` | :white_check_mark: | +| `app/models/project.rb` | `include Avatarable` | **{check-circle}** Yes | +| `app/models/projects/topic.rb` | `include Avatarable` | **{check-circle}** Yes | +| `app/models/group.rb` | `include Avatarable` | **{check-circle}** Yes | +| `app/models/user.rb` | `include Avatarable` | **{check-circle}** Yes | +| `app/models/terraform/state_version.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/ci/job_artifact.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/ci/pipeline_artifact.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/pages_deployment.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/lfs_object.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/dependency_proxy/blob.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/dependency_proxy/manifest.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/packages/composer/cache_file.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/packages/package_file.rb` | `include FileStoreMounter` | **{check-circle}** Yes | +| `app/models/concerns/packages/debian/component_file.rb` | `include FileStoreMounter` | **{check-circle}** Yes | | `ee/app/models/issuable_metric_image.rb` | `include FileStoreMounter` | | | `ee/app/models/vulnerabilities/remediation.rb` | `include FileStoreMounter` | | | `ee/app/models/vulnerabilities/export.rb` | `include FileStoreMounter` | | -| `app/models/packages/debian/project_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | -| `app/models/packages/debian/group_distribution.rb` | `include Packages::Debian::Distribution` | :white_check_mark: | -| `app/models/packages/debian/project_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | -| `app/models/packages/debian/group_component_file.rb` | `include Packages::Debian::ComponentFile` | :white_check_mark: | -| `app/models/merge_request_diff.rb` | `mount_uploader :external_diff, ExternalDiffUploader` | :white_check_mark: | -| `app/models/note.rb` | `mount_uploader :attachment, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :logo, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :header_logo, AttachmentUploader` | :white_check_mark: | -| `app/models/appearance.rb` | `mount_uploader :favicon, FaviconUploader` | :white_check_mark: | +| `app/models/packages/debian/project_distribution.rb` | `include Packages::Debian::Distribution` | **{check-circle}** Yes | +| `app/models/packages/debian/group_distribution.rb` | `include Packages::Debian::Distribution` | **{check-circle}** Yes | +| `app/models/packages/debian/project_component_file.rb` | `include Packages::Debian::ComponentFile` | **{check-circle}** Yes | +| `app/models/packages/debian/group_component_file.rb` | `include Packages::Debian::ComponentFile` | **{check-circle}** Yes | +| `app/models/merge_request_diff.rb` | `mount_uploader :external_diff, ExternalDiffUploader` | **{check-circle}** Yes | +| `app/models/note.rb` | `mount_uploader :attachment, AttachmentUploader` | **{check-circle}** Yes | +| `app/models/appearance.rb` | `mount_uploader :logo, AttachmentUploader` | **{check-circle}** Yes | +| `app/models/appearance.rb` | `mount_uploader :header_logo, AttachmentUploader` | **{check-circle}** Yes | +| `app/models/appearance.rb` | `mount_uploader :favicon, FaviconUploader` | **{check-circle}** Yes | | `app/models/project.rb` | `mount_uploader :bfg_object_map, AttachmentUploader` | | -| `app/models/import_export_upload.rb` | `mount_uploader :import_file, ImportExportUploader` | :white_check_mark: | -| `app/models/import_export_upload.rb` | `mount_uploader :export_file, ImportExportUploader` | :white_check_mark: | +| `app/models/import_export_upload.rb` | `mount_uploader :import_file, ImportExportUploader` | **{check-circle}** Yes | +| `app/models/import_export_upload.rb` | `mount_uploader :export_file, ImportExportUploader` | **{check-circle}** Yes | | `app/models/ci/deleted_object.rb` | `mount_uploader :file, DeletedObjectUploader` | | -| `app/models/design_management/action.rb` | `mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader` | :white_check_mark: | -| `app/models/concerns/packages/debian/distribution.rb` | `mount_uploader :signed_file, Packages::Debian::DistributionReleaseFileUploader` | :white_check_mark: | -| `app/models/bulk_imports/export_upload.rb` | `mount_uploader :export_file, ExportUploader` | :white_check_mark: | +| `app/models/design_management/action.rb` | `mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader` | **{check-circle}** Yes | +| `app/models/concerns/packages/debian/distribution.rb` | `mount_uploader :signed_file, Packages::Debian::DistributionReleaseFileUploader` | **{check-circle}** Yes | +| `app/models/bulk_imports/export_upload.rb` | `mount_uploader :export_file, ExportUploader` | **{check-circle}** Yes | | `ee/app/models/user_permission_export_upload.rb` | `mount_uploader :file, AttachmentUploader` | | | `app/models/ci/secure_file.rb` | `include FileStoreMounter` | | diff --git a/doc/development/workhorse/configuration.md b/doc/development/workhorse/configuration.md index 7f9331e6f1e..ce80a155489 100644 --- a/doc/development/workhorse/configuration.md +++ b/doc/development/workhorse/configuration.md @@ -128,6 +128,25 @@ relative URL in the `authBackend` setting: gitlab-workhorse -authBackend http://localhost:8080/gitlab ``` +## TLS support + +A listener with TLS can be configured to be used for incoming requests. +Paths to the files containing a certificate and matching private key for the server must be provided: + +```toml +[[listeners]] +network = "tcp" +addr = "localhost:3443" +[listeners.tls] + certificate = "/path/to/certificate" + key = "/path/to/private/key" + min_version = "tls1.2" + max_version = "tls1.3" +``` + +The `certificate` file should contain the concatenation +of the server's certificate, any intermediates, and the CA's certificate. + ## Interaction of authBackend and authSocket The interaction between `authBackend` and `authSocket` can be confusing. diff --git a/doc/development/workhorse/index.md b/doc/development/workhorse/index.md index f7ca16e0f31..3aa7e945f53 100644 --- a/doc/development/workhorse/index.md +++ b/doc/development/workhorse/index.md @@ -44,7 +44,7 @@ On some operating systems, such as FreeBSD, you may have to use ### Run time dependencies -Workhorse uses [Exiftool](https://www.sno.phy.queensu.ca/~phil/exiftool/) for +Workhorse uses [ExifTool](https://www.sno.phy.queensu.ca/~phil/exiftool/) for removing EXIF data (which may contain sensitive information) from uploaded images. If you installed GitLab: diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index 2c40efd5909..055cb4c3efb 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -16,11 +16,11 @@ Amazon provides a managed Kubernetes service offering known as [Amazon Elastic K | GitLab Cloud Native Hybrid Ref Arch | GitLab Baseline Perf Test Results Omnibus on Instances | AWS Bill of Materials (BOM) for CNH | AWS Build Performance Testing Results for [CNH](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | CNH Cost Estimate 3 AZs* | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | -| [2K Omnibus](../../administration/reference_architectures/2k_users.md) | [2K Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k) | [2K Cloud Native Hybrid on EKS](#2k-cloud-native-hybrid-on-eks) | GPT Test Results | [1 YR Ec2 Compute Savings + 1 YR RDS & Elasticache RIs](https://calculator.aws/#/estimate?id=544bcf1162beae6b8130ad257d081cdf9d4504e3)<br />(2 AZ Cost Estimate is in BOM Below) | -| [3K](../../administration/reference_architectures/3k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [3k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k) | [3K Cloud Native Hybrid on EKS](#3k-cloud-native-hybrid-on-eks) | [3K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt)<br /><br />[3K Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & Elasticache RIs](https://calculator.aws/#/estimate?id=f1294fec554e21be999711cddcdab9c5e7f83f14)<br />(2 AZ Cost Estimate is in BOM Below) | -| [5K](../../administration/reference_architectures/5k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [5k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/5k) | [5K Cloud Native Hybrid on EKS](#5k-cloud-native-hybrid-on-eks) | [5K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt)<br /><br />[5K AutoScale from 25% GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & Elasticache RIs](https://calculator.aws/#/estimate?id=330ee43c5b14662db5df6e52b34898d181a09e16) | -| [10K](../../administration/reference_architectures/10k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [10k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k) | [10K Cloud Native Hybrid on EKS](#10k-cloud-native-hybrid-on-eks) | [10K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647_results.txt)<br /><br />[10K Elastic Auto Scale GPT Test Results](hhttps://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139_results.txt) | [10K 1 YR Ec2 Compute Savings + 1 YR RDS & Elasticache RIs](https://calculator.aws/#/estimate?id=5ac2e07a22e01c36ee76b5477c5a046cd1bea792) | -| [50K](../../administration/reference_architectures/50k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [50k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/50k) | [50K Cloud Native Hybrid on EKS](#50k-cloud-native-hybrid-on-eks) | [50K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819_results.txt)<br /><br />[10K Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633.txt) | [50K 1 YR Ec2 Compute Savings + 1 YR RDS & Elasticache RIs](https://calculator.aws/#/estimate?id=b9c9d6ac1d4a7848011d2050cef3120931fb7c22) | +| [2K Omnibus](../../administration/reference_architectures/2k_users.md) | [2K Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k) | [2K Cloud Native Hybrid on EKS](#2k-cloud-native-hybrid-on-eks) | GPT Test Results | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=544bcf1162beae6b8130ad257d081cdf9d4504e3)<br />(2 AZ Cost Estimate is in BOM Below) | +| [3K](../../administration/reference_architectures/3k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [3k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k) | [3K Cloud Native Hybrid on EKS](#3k-cloud-native-hybrid-on-eks) | [3K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt)<br /><br />[3K Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200/3k-QuickStart-AutoScale-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_194200_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=f1294fec554e21be999711cddcdab9c5e7f83f14)<br />(2 AZ Cost Estimate is in BOM Below) | +| [5K](../../administration/reference_architectures/5k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [5k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/5k) | [5K Cloud Native Hybrid on EKS](#5k-cloud-native-hybrid-on-eks) | [5K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt)<br /><br />[5K AutoScale from 25% GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717/5k-QuickStart-AutoScale-From-25Percent-ARM-RDS-Redis_v13-12-3-ee_2021-07-24_102717_results.txt) | [1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=330ee43c5b14662db5df6e52b34898d181a09e16) | +| [10K](../../administration/reference_architectures/10k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [10k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k) | [10K Cloud Native Hybrid on EKS](#10k-cloud-native-hybrid-on-eks) | [10K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647_results.txt)<br /><br />[10K Elastic Auto Scale GPT Test Results](hhttps://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139_results.txt) | [10K 1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=5ac2e07a22e01c36ee76b5477c5a046cd1bea792) | +| [50K](../../administration/reference_architectures/50k_users.md#cloud-native-hybrid-reference-architecture-with-helm-charts-alternative) | [50k Baseline](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/50k) | [50K Cloud Native Hybrid on EKS](#50k-cloud-native-hybrid-on-eks) | [50K Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819_results.txt)<br /><br />[10K Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633.txt) | [50K 1 YR Ec2 Compute Savings + 1 YR RDS & ElastiCache RIs](https://calculator.aws/#/estimate?id=b9c9d6ac1d4a7848011d2050cef3120931fb7c22) | \*Cost calculations for actual implementations are a rough guideline with the following considerations: @@ -35,23 +35,24 @@ Amazon provides a managed Kubernetes service offering known as [Amazon Elastic K The [AWS Quick Start for GitLab Cloud Native Hybrid on EKS](https://aws-quickstart.github.io/quickstart-eks-gitlab/) is developed by AWS, GitLab, and the community that contributes to AWS Quick Starts, whether directly to the GitLab Quick Start or to the underlying Quick Start dependencies GitLab inherits (for example, EKS Quick Start). NOTE: -This automation is in **Developer Preview**. GitLab is working with AWS on resolving [the outstanding issues](https://github.com/aws-quickstart/quickstart-eks-gitlab/issues?q=is%3Aissue+is%3Aopen+%5BHL%5D) before it is fully released. You can subscribe to this issue to be notified of progress and release announcements: [AWS Quick Start for GitLab Cloud Native Hybrid on EKS Status: DEVELOPER PREVIEW](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues/11).<br><br> -The developer preview deploys Aurora PostgreSQL, but the release version will deploy Amazon RDS PostgreSQL due to [known issues](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues?label_name%5B%5D=AWS+Known+Issue) with Aurora. All performance testing results will also be redone after this change has been made. +This automation is in **[Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta)**. GitLab is working with AWS on resolving [the outstanding issues](https://github.com/aws-quickstart/quickstart-eks-gitlab/issues?q=is%3Aissue+is%3Aopen+%5BHL%5D) before it is fully released. You can subscribe to this issue to be notified of progress and release announcements: [AWS Quick Start for GitLab Cloud Native Hybrid on EKS Status: Beta](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues/11).<br><br> +The Beta version deploys Aurora PostgreSQL, but the release version will deploy Amazon RDS PostgreSQL due to [known issues](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues?label_name%5B%5D=AWS+Known+Issue) with Aurora. All performance testing results will also be redone after this change has been made. The [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/tree/main) is an effort made by GitLab to create a multi-cloud, multi-GitLab (Omnibus + Cloud Native Hybrid) toolkit to provision GitLab. GET is developed by GitLab developers and is open to community contributions. It is helpful to review the [GitLab Environment Toolkit (GET) Issues](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/issues) to understand if any of them may affect your provisioning plans. | | [AWS Quick Start for GitLab Cloud Native Hybrid on EKS](https://aws-quickstart.github.io/quickstart-eks-gitlab/) | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | -| Licensing | [Open Source (Apache 2.0)](https://github.com/aws-quickstart/quickstart-eks-gitlab/blob/main/LICENSE.txt) | [GitLab Enterprise Edition license](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/LICENSE) ([GitLab Premium tier](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md)) | | Overview and Vision | [AWS Quick Start](https://aws.amazon.com/quickstart/) | [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) | +| Licensing | [Open Source (Apache 2.0)](https://github.com/aws-quickstart/quickstart-eks-gitlab/blob/main/LICENSE.txt) | [GitLab Enterprise Edition license](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/LICENSE) ([GitLab Premium tier](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md)) | +| GitLab Support | [GitLab Beta Support](../../policy/alpha-beta-support.md#beta-features) | [GitLab GA Support](../../policy/alpha-beta-support.md#generally-available-ga) | | GitLab Reference Architecture Compliant | Yes | Yes | | GitLab Performance Tool (GPT) Tested | Yes | Yes | | Amazon Well Architected Compliant | Yes<br />(via Quick Start program) | Critical portions <br />reviewed by AWS | | Target Cloud Platforms | AWS | AWS, Google, Azure | | IaC Languages | CloudFormation (Quick Starts) | Terraform, Ansible | | Community Contributions and Participation (EcoSystem) | <u>GitLab QSG</u>: Getting Started<br /><u>For QSG Dependencies (for example, EKS)</u>: Substantial | Getting Started | -| Compatible with AWS Meta-Automation Services (via CloudFormation) | - [AWS Service Catalog](https://aws.amazon.com/servicecatalog/) (Direct Import)<br>- [ServiceNow via an AWS Service Catalog Connector](https://docs.aws.amazon.com/servicecatalog/latest/adminguide/integrations-servicenow.html#integrations-servicenow)<br>- [Jira Service Manager via an AWS Service Catalog Connector](https://docs.aws.amazon.com/servicecatalog/latest/adminguide/integrations-jiraservicedesk.html#integrations-jiraservicedesk)<br>- [AWS Control Tower](https://docs.aws.amazon.com/controltower/) ([Integration](https://aws.amazon.com/blogs/infrastructure-and-automation/deploy-aws-quick-start-to-multiple-accounts-using-aws-control-tower/))<br>- Quick Starts<br>- [AWS SaaS Factory](https://aws.amazon.com/partners/programs/saas-factory/) | No | +| Compatible with AWS Meta-Automation Services (via CloudFormation) | - [AWS Service Catalog](https://aws.amazon.com/servicecatalog/) (Direct Import)<br>- [ServiceNow via an AWS Service Catalog Connector](https://docs.aws.amazon.com/servicecatalog/latest/adminguide/integrations-servicenow.html#integrations-servicenow)<br>- [Jira Service Manager via an AWS Service Catalog Connector](https://docs.aws.amazon.com/servicecatalog/latest/adminguide/integrations-jiraservicedesk.html#integrations-jiraservicedesk)<br>- [AWS Control Tower](https://docs.aws.amazon.com/controltower/) ([Integration](https://aws.amazon.com/blogs/infrastructure-and-automation/deploy-aws-quick-start-to-multiple-accounts-using-aws-control-tower/))<br>- Quick Starts<br>- [AWS SaaS Factory](https://aws.amazon.com/partners/programs/saas-factory/) | No | | Results in a Ready-to-Use instance | Yes | Manual Actions or <br />Supplemental IaC Required | | **<u>Configuration Features</u>** | | | | Can deploy Omnibus GitLab (non-Kubernetes) | No | Yes | @@ -98,7 +99,7 @@ Some services, such as log aggregation, outbound email are not specified by GitL | ------------------------------------------------------------ | ------------------------------ | ------------------------------------------------------------ | | <u>Tested PaaS Mentioned in Reference Architectures</u> | | | | **PostgreSQL Database** | Amazon RDS PostgreSQL | Yes. | -| **Redis Caching** | Redis Elasticache | Yes. | +| **Redis Caching** | Redis ElastiCache | Yes. | | **Gitaly Cluster (Git Repository Storage)**<br />(Including Praefect and PostgreSQL) | ASG and Instances | Yes - ASG and Instances<br />**Note: Gitaly cannot be put into a Kubernetes Cluster.** | | **All GitLab storages besides Git Repository Storage**<br />(Includes Git-LFS which is S3 Compatible) | AWS S3 | Yes | | | | | @@ -152,7 +153,7 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | 2 vCPU, 8 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 16 vCPU, 32 GB | | | | One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16GB | | | -| **Grand Total w/ Overheads**<br />Minimum hosts = 3 | 24 vCPU, 48 GB | **c5.2xlarge** <br />(8vcpu/16GB) x 3 nodes<br />24 vCPU, 48 GB | $1.02/hr | +| **Grand Total w/ Overheads**<br />Minimum hosts = 3 | 24 vCPU, 48 GB | **c5.2xlarge** <br />(8vCPU/16GB) x 3 nodes<br />24 vCPU, 48 GB | $1.02/hr | | **Idle Configuration (Scaled-In)** | 16 vCPU, 32 GB | **c5.2xlarge** x 2 | $0.68/hr | NOTE: @@ -205,7 +206,7 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/3k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/3k_users.md#cluster-topology)) = <br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 32 vCPU, 56 GB | | | | One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 16 vCPU, 32GB | | | -| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 48 vCPU, 88 GB | **c5.2xlarge** (8vcpu/16GB) x 5 nodes<br />40 vCPU, 80 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt) | $1.70/hr | +| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 48 vCPU, 88 GB | **c5.2xlarge** (8vCPU/16GB) x 5 nodes<br />40 vCPU, 80 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/3K/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216/3k-QuickStart-ARM-RDS-Cache_v13-12-3-ee_2021-07-23_124216_results.txt) | $1.70/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 24 vCPU, 48 GB | c5.2xlarge x 4 | $1.36/hr | Other combinations of node type and quantity can be used to meet the Grand Total. Due to the properties of pods, hosts that are overly small may have significant unused capacity. @@ -259,10 +260,10 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/5k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/5k_users.md#cluster-topology)) = <br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 62 vCPU, 96.5 GB | | | | One Node for Quick Start Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 8 vCPU, 16GB | | | -| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 70 vCPU, 112.5 GB | **c5.2xlarge** (8vcpu/16GB) x 9 nodes<br />72 vCPU, 144 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | $2.38/hr | +| **Grand Total w/ Overheads Full Scale**<br />Minimum hosts = 3 | 70 vCPU, 112.5 GB | **c5.2xlarge** (8vCPU/16GB) x 9 nodes<br />72 vCPU, 144 GB<br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/5K/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128/5k-QuickStart-ARM-RDS-Redis_v13-12-3-ee_2021-07-23_140128_results.txt) | $2.38/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 24 vCPU, 48 GB | c5.2xlarge x 7 | $1.85/hr | -Other combinations of node type and quantity can be used to meet the Grand Total. Due to the cpu and memory requirements of pods, hosts that are overly small may have significant unused capacity. +Other combinations of node type and quantity can be used to meet the Grand Total. Due to the CPU and memory requirements of pods, hosts that are overly small may have significant unused capacity. NOTE: If EKS node autoscaling is employed, it is likely that your average loading will run lower than this, especially during non-working hours and weekends. @@ -312,10 +313,10 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/10k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/10k_users.md#cluster-topology))<br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 128 vCPU, 158 GB | | | | One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 16 vCPU, 32GB | | | -| **Grand Total w/ Overheads Fully Scaled**<br />Minimum hosts = 3 | 142 vCPU, 190 GB | **c5.4xlarge** (16vcpu/32GB) x 9 nodes<br />144 vCPU, 288GB<br /><br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647_results.txt) | $6.12/hr | +| **Grand Total w/ Overheads Fully Scaled**<br />Minimum hosts = 3 | 142 vCPU, 190 GB | **c5.4xlarge** (16vCPU/32GB) x 9 nodes<br />144 vCPU, 288GB<br /><br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647/GL-CloudNative-10k-RDS-Graviton_v13-12-3-ee_2021-07-08_194647_results.txt) | $6.12/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 40 vCPU, 80 GB | c5.4xlarge x 7<br /><br />[Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/10K/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139/GL-CloudNative-10k-AutoScaling-Test_v13-12-3-ee_2021-07-09_115139_results.txt) | $4.76/hr | -Other combinations of node type and quantity can be used to meet the Grand Total. Due to the cpu and memory requirements of pods, hosts that are overly small may have significant unused capacity. +Other combinations of node type and quantity can be used to meet the Grand Total. Due to the CPU and memory requirements of pods, hosts that are overly small may have significant unused capacity. NOTE: If EKS node autoscaling is employed, it is likely that your average loading will run lower than this, especially during non-working hours and weekends. @@ -365,10 +366,10 @@ On Demand pricing is used in this table for comparisons, but should not be used | Supporting services such as NGINX, Prometheus, etc | [2 allocations](../../administration/reference_architectures/10k_users.md#cluster-topology) x ([2 vCPU and 7.5 GB](../../administration/reference_architectures/10k_users.md#cluster-topology))<br />4 vCPU, 15 GB | | | | **GitLab Ref Arch Raw Total K8s Node Capacity** | 428 vCPU, 533 GB | | | | One Node for Overhead and Miscellaneous (EKS Cluster AutoScaler, Grafana, Prometheus, etc) | + 16 vCPU, 32GB | | | -| **Grand Total w/ Overheads Fully Scaled**<br />Minimum hosts = 3 | 444 vCPU, 565 GB | **c5.4xlarge** (16vcpu/32GB) x 28 nodes<br />448 vCPU, 896GB<br /><br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819_results.txt) | $19.04/hr | +| **Grand Total w/ Overheads Fully Scaled**<br />Minimum hosts = 3 | 444 vCPU, 565 GB | **c5.4xlarge** (16vCPU/32GB) x 28 nodes<br />448 vCPU, 896GB<br /><br />[Full Fixed Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819/50k-Fixed-Scale-Test_v13-12-3-ee_2021-08-13_172819_results.txt) | $19.04/hr | | **Possible Idle Configuration (Scaled-In 75% - round up)**<br />Pod autoscaling must be also adjusted to enable lower idling configuration. | 40 vCPU, 80 GB | c5.2xlarge x 10<br /><br />[Elastic Auto Scale GPT Test Results](https://gitlab.com/guided-explorations/aws/implementation-patterns/gitlab-cloud-native-hybrid-on-eks/-/blob/master/gitlab-alliances-testing/50K/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633/50k-AutoScale-Test_v13-12-3-ee_2021-08-13_192633.txt) | $6.80/hr | -Other combinations of node type and quantity can be used to meet the Grand Total. Due to the cpu and memory requirements of pods, hosts that are overly small may have significant unused capacity. +Other combinations of node type and quantity can be used to meet the Grand Total. Due to the CPU and memory requirements of pods, hosts that are overly small may have significant unused capacity. NOTE: If EKS node autoscaling is employed, it is likely that your average loading will run lower than this, especially during non-working hours and weekends. diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 8dbda7420b0..ee7279d72cd 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -63,7 +63,7 @@ When deploying a GitLab instance using the official AMI, the root password to th Instances running on Community Edition (CE) require a migration to Enterprise Edition (EE) in order to subscribe to the GitLab Premium or Ultimate plan. If you want to pursue a subscription, using the Free-forever plan of Enterprise Edition is the least disruptive method. NOTE: -Since any given GitLab upgrade might involve data disk updates or database schema upgrades, simply swapping out the AMI is not sufficent for taking upgrades. +Since any given GitLab upgrade might involve data disk updates or database schema upgrades, simply swapping out the AMI is not sufficient for taking upgrades. 1. Log in to the AWS Web Console, so that clicking the links in the following step take you directly to the AMI list. 1. Pick the edition you want: diff --git a/doc/install/docker.md b/doc/install/docker.md index a25ed629681..d5ca2c2e29f 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -189,7 +189,7 @@ services: external_url 'http://gitlab.example.com:8929' gitlab_rails['gitlab_shell_ssh_port'] = 2224 ports: - - '8929:80' + - '8929:8929' - '2224:22' volumes: - '$GITLAB_HOME/config:/etc/gitlab' @@ -198,7 +198,7 @@ services: shm_size: '256m' ``` -This is the same as using `--publish 8929:80 --publish 2224:22`. +This is the same as using `--publish 8929:8929 --publish 2224:22`. ### Install GitLab using Docker swarm mode @@ -257,7 +257,7 @@ Here's an example that deploys GitLab with four runners as a [stack](https://doc ```ruby external_url 'https://my.domain.com/' - gitlab_rails['initial_root_password'] = File.read('/run/secrets/gitlab_root_password') + gitlab_rails['initial_root_password'] = File.read('/run/secrets/gitlab_root_password').gsub("\n", "") ``` 1. Create a `root_password.txt` file: @@ -664,7 +664,7 @@ writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with un writing value to /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-0.db failed with unmapped file ``` -Other than disabling the Prometheus Metrics from the Admin page, the recommended +Other than disabling the Prometheus Metrics from the Admin Area, the recommended solution to fix this problem is to increase the size of shared memory to at least 256MB. If using `docker run`, this can be done by passing the flag `--shm-size 256m`. If using a `docker-compose.yml` file, the `shm_size` key can be used for this diff --git a/doc/install/index.md b/doc/install/index.md index deb94031e44..52bc9062adc 100644 --- a/doc/install/index.md +++ b/doc/install/index.md @@ -28,10 +28,10 @@ install GitLab: | Installation method | Description | When to choose | |----------------------------------------------------------------|-------------|----------------| | [Linux package](https://docs.gitlab.com/omnibus/installation/) | The official deb/rpm packages (also known as Omnibus GitLab) that contains a bundle of GitLab and the components it depends on, including PostgreSQL, Redis, and Sidekiq. | This method is recommended for getting started. The Linux packages are mature, scalable, and are used today on GitLab.com. If you need additional flexibility and resilience, we recommend deploying GitLab as described in the [reference architecture documentation](../administration/reference_architectures/index.md). | -| [Helm charts](https://docs.gitlab.com/charts/) | The cloud native Helm chart for installing GitLab and all of its components on Kubernetes. | When installing GitLab on Kubernetes, there are some trade-offs that you need to be aware of: <br/>- Administration and troubleshooting requires Kubernetes knowledge.<br/>- It can be more expensive for smaller installations. The default installation requires more resources than a single node Linux package deployment, as most services are deployed in a redundant fashion.<br/><br/> Use this method if your infrastructure is built on Kubernetes and you're familiar with how it works. The methods for management, observability, and some concepts are different than traditional deployments. | +| [Helm charts](https://docs.gitlab.com/charts/) | The cloud native Helm chart for installing GitLab and all of its components on Kubernetes. | When installing GitLab on Kubernetes, it has some trade-offs that you must be aware of: <br/>- Administration and troubleshooting requires Kubernetes knowledge.<br/>- It can be more expensive for smaller installations. The default installation requires more resources than a single node Linux package deployment, as most services are deployed in a redundant fashion.<br/><br/> Use this method if your infrastructure is built on Kubernetes and you're familiar with how it works. The methods for management, observability, and some concepts are different than traditional deployments. | | [Docker](docker.md) | The GitLab packages, Dockerized. | Use this method if you're familiar with Docker. | -| [Source](installation.md) | Install GitLab and all of its components from scratch. | Use this method if none of the previous methods are available for your platform. Useful for unsupported systems like \*BSD.| -| [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#documentation) | The GitLab Environment Toolkit provides a set of automation tools to deploy a [reference architecture](../administration/reference_architectures/index.md) on most major cloud providers. | Customers are very welcome to trial and evaluate GET today, however be aware of [key limitations](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#missing-features-to-be-aware-of) of the current iteration. For production environments further manual setup will be required based on your specific requirements. | +| [Source](installation.md) | Install GitLab and all of its components from scratch. | Use this method if none of the previous methods are available for your platform. Can be used for unsupported systems like \*BSD.| +| [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#documentation) | The GitLab Environment toolkit provides a set of automation tools to deploy a [reference architecture](../administration/reference_architectures/index.md) on most major cloud providers. | Customers are very welcome to trial and evaluate GET today, however be aware of [key limitations](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#missing-features-to-be-aware-of) of the current iteration. For production environments further manual setup is required based on your specific requirements. | | [GitLab Operator](https://docs.gitlab.com/operator/) | The GitLab Operator provides an installation and management method for GitLab following the [Kubernetes Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/). | Use the GitLab Operator to run GitLab in an [OpenShift](openshift_and_gitlab/index.md) environment. | ## Install GitLab on cloud providers diff --git a/doc/install/installation.md b/doc/install/installation.md index c03c600fe0b..e53ced6c88a 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -197,13 +197,9 @@ can install it with: sudo apt-get install -y postfix ``` -Then select 'Internet Site' and press enter to confirm the hostname. +Then select 'Internet Site' and press <kbd>Enter</kbd> to confirm the hostname. -<!-- vale gitlab.Spelling = NO --> - -### Exiftool - -<!-- vale gitlab.Spelling = YES --> +### ExifTool [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse#dependencies) requires `exiftool` to remove EXIF data from uploaded images. @@ -230,9 +226,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.4.tar.gz" -echo '3043099089608859fc8cce7f9fdccaa1f53a462457e3838ec3b25a7d609fbc5b ruby-2.7.4.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.4.tar.gz -cd ruby-2.7.4 +curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz" +echo '2755b900a21235b443bb16dadd9032f784d4a88f143d852bc5d154f22b8781f1 ruby-2.7.5.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.5.tar.gz +cd ruby-2.7.5 ./configure --disable-install-rdoc --enable-shared make @@ -1049,13 +1045,6 @@ follow the normal directions and generate a self-signed SSL certificate: sudo chmod o-r gitlab.key ``` -WARNING: -The `self_signed_cert` variable is -[deprecated and redundant](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/120). -It is set to `false` by default, but still accepts self-signed certificates. Setting -this value to `true` allows any certificate to be accepted, and can make -machine-in-the-middle attacks possible. - ### Enable Reply by email See the ["Reply by email" documentation](../administration/reply_by_email.md) for more information on how to set this up. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 8006b886414..83fce2f00f6 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -121,6 +121,8 @@ the following table) as these were used for development and testing: |----------------|----------------------------| | 13.0 | 11 | | 14.0 | 12.10 | +| 15.0 | 12.0 | +| 16.0 (planned) | 13.0 | You must also ensure the following extensions are loaded into every GitLab database. [Read more about this requirement, and troubleshooting](postgresql_extensions.md). @@ -132,7 +134,7 @@ GitLab database. [Read more about this requirement, and troubleshooting](postgre | `plpgsql` | 11.7 | NOTE: -Support for [PostgreSQL 9.6 and 10 was removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). +Support for [PostgreSQL 9.6 and 10 was removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. #### Additional requirements for GitLab Geo diff --git a/doc/integration/akismet.md b/doc/integration/akismet.md index 5541c5914d5..9a85511836f 100644 --- a/doc/integration/akismet.md +++ b/doc/integration/akismet.md @@ -12,7 +12,7 @@ Akismet for review, and instance administrators can [mark snippets as spam](../user/snippets.md#mark-snippet-as-spam). Detected spam is rejected, and an entry is added in the **Spam Log** section of the -Admin page. +Admin Area. Privacy note: GitLab submits the user's IP and user agent to Akismet. diff --git a/doc/integration/alicloud.md b/doc/integration/alicloud.md index 85e1e2d4154..1619bdc9504 100644 --- a/doc/integration/alicloud.md +++ b/doc/integration/alicloud.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -28,7 +28,7 @@ Sign in to the AliCloud platform and create an application on it. AliCloud gener Select **Save**. 1. Add OAuth scopes in the application details page: - + 1. Under the **Application Name** column, select the name of the application you created. The application's details page opens. 1. Under the **Application OAuth Scopes** tab, select **Add OAuth Scopes**. 1. Select the **aliuid** and **profile** checkboxes. diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index 8bac95c5f04..1e1ee9aebc5 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/azure.md b/doc/integration/azure.md index 5749e638164..515e7406545 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 2fc80aa1769..165d25c0d90 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -124,4 +124,4 @@ After the above configuration is set up, you can use Bitbucket to sign into GitLab and [start importing your projects](../user/project/import/bitbucket.md). If you want to import projects from Bitbucket, but don't want to enable signing in, -you can [disable Sign-Ins in the admin panel](omniauth.md#enable-or-disable-sign-in-with-an-omniauth-provider-without-disabling-import-sources). +you can [disable Sign-Ins in the Admin Area](omniauth.md#enable-or-disable-sign-in-with-an-omniauth-provider-without-disabling-import-sources). diff --git a/doc/integration/cas.md b/doc/integration/cas.md index 9594836164a..a0cb6bd98cd 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/ding_talk.md b/doc/integration/ding_talk.md index 2bc4f29298a..437648b1adf 100644 --- a/doc/integration/ding_talk.md +++ b/doc/integration/ding_talk.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 5197a191b8e..214bd845944 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -217,7 +217,7 @@ The following Elasticsearch settings are available: | `Password` | The password of your Elasticsearch instance. | | `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indexes with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | | `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | -| `Limit namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-namespaces-and-projects). +| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). | `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | | `AWS Region` | The AWS region in which your OpenSearch Service is located. | | `AWS Access Key` | The AWS access key. | @@ -234,9 +234,10 @@ Sidekiq performance. Return them to their default values if you see increased `s in your Sidekiq logs. For more information, see [issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). -### Limit namespaces and projects +### Limit the number of namespaces and projects that can be indexed -If you select `Limit namespaces and projects that can be indexed`, more options become available. +If you check checkbox `Limit the number of namespaces and projects that can be indexed` +under **Elasticsearch indexing restrictions** more options become available.  @@ -871,7 +872,7 @@ There are a couple of ways to achieve that: This is always correctly identifying whether the current project/namespace being searched is using Elasticsearch. -- From the admin area under **Settings > Advanced Search** check that the +- From the Admin Area under **Settings > Advanced Search** check that the Advanced Search settings are checked. Those same settings there can be obtained from the Rails console if necessary: @@ -920,7 +921,7 @@ More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticse It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../administration/troubleshooting/elasticsearch.md#search-results-workflow). NOTE: -The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limit-namespaces-and-projects). +The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). See [Elasticsearch Index Scopes](#advanced-search-index-scopes) for more information on searching for specific types of data. @@ -1082,7 +1083,7 @@ If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error duri ### Indexing is very slow or fails with `rejected execution of coordinating operation` messages -Bulk requests are getting rejected by the Elasticsearch node(s) likely due to load and lack of available memory. +Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. Ensure that your Elasticsearch cluster meets the [system requirements](#system-requirements) and has enough resources to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index b94fa24d290..c73a8d5e461 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/github.md b/doc/integration/github.md index a265d5c67ed..3c14e8db4de 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 132006ab996..68ba676b539 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/gitpod.md b/doc/integration/gitpod.md index f54542ff43f..84294a99ef9 100644 --- a/doc/integration/gitpod.md +++ b/doc/integration/gitpod.md @@ -61,7 +61,7 @@ Your users can then [enable it for themselves](#enable-gitpod-in-your-user-setti You can launch Gitpod directly from GitLab in one of these ways: - *From your project's page:* - 1. Go to your project, and navigate to the page you want to edit. + 1. Go to your project, then go to the page you want to edit. 1. Select the caret **(angle-down)** next to **Web IDE**, and select **Gitpod** from the list: @@ -69,13 +69,7 @@ You can launch Gitpod directly from GitLab in one of these ways: 1. Select **Open in Gitpod**. - *From a merge request:* - 1. Go to your merge request, and select **Overview** in the tab menu. - 1. Scroll to the end of the merge request description. - 1. Select the caret **(angle-down)** next to **Web IDE**, then select **Gitpod** - from the list: - -  - - 1. Select **Open in Gitpod**. + 1. Go to your merge request. + 1. In the upper right corner, select **Code**, then select **Open in Gitpod**. Gitpod builds your development environment for your branch. diff --git a/doc/integration/google.md b/doc/integration/google.md index d6a37dbf30f..596700822cd 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/img/gitpod-button_v14_2.png b/doc/integration/img/gitpod-button_v14_2.png Binary files differdeleted file mode 100644 index 04dd5b50a7f..00000000000 --- a/doc/integration/img/gitpod-button_v14_2.png +++ /dev/null diff --git a/doc/integration/img/limit_namespaces_projects_options.png b/doc/integration/img/limit_namespaces_projects_options.png Binary files differindex fa666c7491e..a3cf9933b7e 100644 --- a/doc/integration/img/limit_namespaces_projects_options.png +++ b/doc/integration/img/limit_namespaces_projects_options.png diff --git a/doc/integration/index.md b/doc/integration/index.md index b26c841f943..f1d16dc409d 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -64,7 +64,7 @@ or [Kroki](../administration/integration/kroki.md) to use diagrams in AsciiDoc a ## Integrations -Integration with services such as Campfire, Flowdock, Jira, Pivotal Tracker, and Slack are available as [Integrations](../user/project/integrations/overview.md). +Integration with services such as Campfire, Flowdock, Jira, Pivotal Tracker, and Slack are available as [Integrations](../user/project/integrations/index.md). ## Troubleshooting diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 54d235b5357..7b02580dac1 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -137,7 +137,7 @@ than the [webhook integration](#configure-a-webhook). - Merge request - Tag push 1. Enter the **Jenkins server URL**. -1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](../user/project/integrations/overview.md#ssl-verification). +1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](../user/project/integrations/index.md#manage-ssl-verification). 1. Enter the **Project name**. The project name should be URL-friendly, where spaces are replaced with underscores. To ensure @@ -201,7 +201,7 @@ This issue can occur when the request exceeds the [webhook timeout](../user/project/integrations/webhooks.md#webhook-fails-or-multiple-webhook-requests-are-triggered), which is set to 10 seconds by default. -Check the [service hook logs](../user/project/integrations/overview.md#troubleshooting-integrations) +Check the [service hook logs](../user/project/integrations/index.md#troubleshooting-integrations) for request failures or check the `/var/log/gitlab/gitlab-rails/production.log` file for messages like: diff --git a/doc/integration/jira/issues.md b/doc/integration/jira/issues.md index 28998851697..aaff5de767b 100644 --- a/doc/integration/jira/issues.md +++ b/doc/integration/jira/issues.md @@ -138,6 +138,10 @@ of these filters: Enhancements to use these filters through the user interface [are planned](https://gitlab.com/groups/gitlab-org/-/epics/3622). +## Create a Jira issue for a vulnerability **(ULTIMATE)** + +You can create a Jira issue for a vulnerability from a [Vulnerability Page](../../user/application_security/vulnerabilities/index.md#create-a-jira-issue-for-a-vulnerability). + ## Automatic issue transitions > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55773) in GitLab 13.11. diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 17a81419ad0..73674b66f05 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -299,9 +299,11 @@ Kerberos ticket-based authentication. ## Upgrading from password-based to ticket-based Kerberos sign-ins In previous versions of GitLab users had to submit their -Kerberos username and password to GitLab when signing in. We plan to -remove support for password-based Kerberos sign-ins in a future -release, so we recommend that you upgrade to ticket-based sign-ins. +Kerberos username and password to GitLab when signing in. + +We [deprecated](../update/deprecations.md#omniauth-kerberos-gem) password-based +Kerberos sign-ins in GitLab 14.3 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/2908) +it in GitLab 15.0. You must switch to ticket-based sign in. Depending on your existing GitLab configuration, the 'Sign in with: Kerberos SPNEGO' button may already be visible on your GitLab sign-in diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index c8e2df1f88f..5ea723abba9 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -95,6 +95,7 @@ mattermost_external_url 'http://mattermost.example.com' gitlab_rails['enable'] = false redis['enable'] = false postgres_exporter['enable'] = false +grafana['enable'] = false ``` Then follow the appropriate steps in the [Authorize GitLab Mattermost section](#authorize-gitlab-mattermost). Last, to enable diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index 3d44da8b4c8..e3ec1aa16a1 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/oauth_provider.md b/doc/integration/oauth_provider.md index adfb2fad941..e33f874d35a 100644 --- a/doc/integration/oauth_provider.md +++ b/doc/integration/oauth_provider.md @@ -88,11 +88,13 @@ The user authorization step is automatically skipped for this application. ## Expiring access tokens -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21745) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21745) in GitLab 14.3, with the ability to opt out. +> - Ability to opt-out of expiring access token [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/340848) in GitLab 15.0. WARNING: -The ability to opt-out of expiring access tokens [is deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/340848). -All existing integrations should be updated to support access token refresh. +The ability to opt-out of expiring access tokens was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/340848) +in GitLab 14.3 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/340848) in 15.0. All +existing integrations must be updated to support access token refresh. Access tokens expire in two hours which means that integrations that use them must support generating new access tokens at least every two hours. Existing: diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index f6e41e808af..296de9e511a 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/openid_connect_provider.md b/doc/integration/openid_connect_provider.md index 1254b29bfb9..cc9c8ffd012 100644 --- a/doc/integration/openid_connect_provider.md +++ b/doc/integration/openid_connect_provider.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 8d4d8ff9f52..67c24415226 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/integration/saml.md b/doc/integration/saml.md index c5383f9e34b..ee4c34bb924 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -391,7 +391,7 @@ The requirements are the same as the previous settings: - The IdP must pass Group information to GitLab. - GitLab should know where to look for the groups in the SAML response, as well as which - group(s) entail that a given user is an [auditor user](../user/permissions.md#auditor-users). + groups include users with the [Auditor role](../user/permissions.md#auditor-users). ```yaml { name: 'saml', @@ -853,7 +853,7 @@ When configuring the Google Workspace SAML app, be sure to record the following | SSO URL | Depends | Google Identity Provider details. Set to the GitLab `idp_sso_target_url` setting. | | Certificate | Downloadable | Run `openssl x509 -in <your_certificate.crt> -noout -fingerprint` to generate the SHA1 fingerprint that can be used in the `idp_cert_fingerprint` setting. | -While the Google Workspace Admin provides IdP metadata, Entity ID, and SHA-256 +While the Google Workspace Administrator provides IdP metadata, Entity ID, and SHA-256 fingerprint, they are not required. GitLab does not need that information to connect to the Google Workspace SAML app. diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index b2e5f7b4b7d..72ad0bcc32d 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -41,7 +41,7 @@ If you are using an HTTPS connection to GitLab, you must [configure HTTPS](https ### Connect your Sourcegraph instance to your GitLab instance -1. Navigate to the site admin area in Sourcegraph. +1. Navigate to the site Admin Area in Sourcegraph. 1. [Configure your GitLab external service](https://docs.sourcegraph.com/admin/external_service/gitlab). You can skip this step if you already have your GitLab repositories searchable in Sourcegraph. 1. Validate that you can search your repositories from GitLab in your Sourcegraph instance by running a test query. diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index 3aba6b70b94..a7facf76f05 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Manage +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md index 907c59adacb..2a007eade99 100644 --- a/doc/operations/error_tracking.md +++ b/doc/operations/error_tracking.md @@ -1,6 +1,6 @@ --- stage: Monitor -group: Respond +group: Observability info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -32,7 +32,7 @@ For error tracking to work, you need two pieces: ### Deploying Sentry -You can sign up to the cloud hosted [Sentry](https://sentry.io), deploy your own [on-premise instance](https://github.com/getsentry/onpremise/), or use GitLab to [install Sentry to a Kubernetes cluster](../user/clusters/applications.md#install-sentry-using-gitlab-cicd). +You can sign up to the cloud hosted [Sentry](https://sentry.io), deploy your own [on-premise instance](https://github.com/getsentry/onpremise/), or use GitLab to [install Sentry to a Kubernetes cluster](../user/infrastructure/clusters/manage/management_project_applications/sentry.md). ### Enabling Sentry diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index e7d78beb0b9..4b503af2cf7 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -402,7 +402,7 @@ docker run \ > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/251234) in GitLab 13.5. > - Showing related feature flags in issues [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220333) in GitLab 14.1. -You can link related issues to a feature flag. In the **Linked issues** section, +You can link related issues to a feature flag. In the Feature Flag **Linked issues** section, click the `+` button and input the issue reference number or the full URL of the issue. The issues then appear in the related feature flag and the other way round. @@ -430,7 +430,7 @@ Please note that the polling rate is configurable in SDKs. Provided that all cli For applications looking for more scalable solution, we recommend to use [Unleash Proxy](#unleash-proxy-example). This proxy server sits between the server and clients. It requests to the server as a behalf of the client groups, -so the nubmer of outbound requests can be greatly reduced. +so the number of outbound requests can be greatly reduced. There is also an [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/295472) to give more capacity to the current rate limit. diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md index 5eb0624be31..008e41f5d64 100644 --- a/doc/operations/incident_management/alerts.md +++ b/doc/operations/incident_management/alerts.md @@ -202,7 +202,7 @@ To assign an alert: 1. Select your desired alert to display its details. -  +  1. If the right sidebar is not expanded, select **Expand sidebar** (**{angle-double-right}**) to expand it. diff --git a/doc/operations/incident_management/paging.md b/doc/operations/incident_management/paging.md index d0c14ab6e59..420454d8d8a 100644 --- a/doc/operations/incident_management/paging.md +++ b/doc/operations/incident_management/paging.md @@ -38,7 +38,7 @@ a single email notification for new alerts. ## Paging **(PREMIUM)** -In projects that have an [escalation policy](escalation_policies.md) configured, on-call responder(s) +In projects that have an [escalation policy](escalation_policies.md) configured, on-call responders can be automatically paged about critical problems through email. ### Escalating an alert diff --git a/doc/operations/index.md b/doc/operations/index.md index 61d38c3d6a9..88687c2faf1 100644 --- a/doc/operations/index.md +++ b/doc/operations/index.md @@ -11,11 +11,11 @@ your applications. ## Measure reliability and stability with metrics (DEPRECATED) -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 14.7. +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) -in GitLab 14.7, and is planned for removal in GitLab 15.0. +This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) +for use in GitLab 14.7, and is planned for removal in GitLab 16.0. Metrics help you understand the health and performance of your infrastructure, applications, and systems by providing insights into your application's reliability, @@ -77,13 +77,20 @@ microservices-based distributed systems - and displays results within GitLab. - [Trace the performance and health](tracing.md) of a deployed application. -## Aggregate and store logs (DEPRECATED) +## Aggregate and store logs (DEPRECATED) **(FREE SELF)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 14.7. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 14.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/360182) behind a [feature flag](../administration/feature_flags.md) named `monitor_logging` in GitLab 15.0. Disabled by default. WARNING: This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) -in GitLab 14.7, and is planned for removal in GitLab 15.0. +in GitLab 14.7. +It will be removed completely in GitLab 15.2. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `monitor_logging`. +On GitLab.com, this feature is not available. +This feature is not recommended for production use. Developers need to troubleshoot application changes in development, and incident responders need aggregated, real-time logs when troubleshooting problems with diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md index 26ea7aa07eb..44b998dc3ee 100644 --- a/doc/operations/metrics/embed.md +++ b/doc/operations/metrics/embed.md @@ -6,6 +6,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Embedding metric charts within GitLab Flavored Markdown **(FREE)** +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. + +WARNING: +This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) +for use in GitLab 14.7, and is planned for removal in GitLab 16.0. + You can display metrics charts within [GitLab Flavored Markdown (GLFM)](../../user/markdown.md) fields such as issue or merge request descriptions. The maximum number of embedded diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index b04e19807f8..937bd329570 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -6,6 +6,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Monitor your environment's metrics **(FREE)** +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. + +WARNING: +This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) +for use in GitLab 14.7, and is planned for removal in GitLab 16.0. + GitLab helps your team monitor the health and performance of your applications and infrastructure by turning statistics and log files into charts and graphs that are easy to understand, especially when time is short and decisions are @@ -116,7 +122,7 @@ dashboard is visible to authenticated and non-authenticated users. Custom metrics can be monitored by adding them on the monitoring dashboard page. After saving them, they display on the environment metrics dashboard provided that either: -- A [connected Kubernetes cluster](../../user/project/clusters/add_remove_clusters.md) +- A [connected Kubernetes cluster](../../user/clusters/agent/index.md) with the matching [environment scope](../../ci/environments/index.md#scope-environments-with-specs) is used and [Prometheus installed on the cluster](../../user/project/integrations/prometheus.md#enabling-prometheus-integration). - Prometheus is [manually configured](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md index bbc05bb0f9a..e68d7077328 100644 --- a/doc/operations/tracing.md +++ b/doc/operations/tracing.md @@ -4,14 +4,21 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Tracing (DEPRECATED) **(FREE)** +# Tracing (DEPRECATED) **(FREE SELF)** > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42645) from GitLab Ultimate to GitLab Free in 13.5. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346540) in GitLab 14.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/359904) behind a [feature flag](../administration/feature_flags.md) named `monitor_tracing` in GitLab 15.0. Disabled by default. WARNING: This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346540) -in GitLab 14.7, and is planned for removal in GitLab 15.0. +in GitLab 14.7. +It will be removed completely in GitLab 15.2. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `monitor_tracing`. +On GitLab.com, this feature is not available. +This feature is not recommended for production use. Tracing provides insight into the performance and health of a deployed application, tracking each function or microservice that handles a given request. Tracing makes it easy to understand the diff --git a/doc/policy/alpha-beta-support.md b/doc/policy/alpha-beta-support.md index ba988b38af2..1e3b591735c 100644 --- a/doc/policy/alpha-beta-support.md +++ b/doc/policy/alpha-beta-support.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w <!-- any changes made to this page should be reflected in https://about.gitlab.com/support/statement-of-support.html#alpha--beta-features and https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga --> -# Support for Alpha, Beta, and Generally Available Features **(PREMIUM)** +# Support for Alpha, Beta, Limited Availability, and Generally Available Features **(PREMIUM)** Some GitLab features are released as [Alpha or Beta versions](https://about.gitlab.com/support/statement-of-support.html#alpha--beta-features) which are not fully supported. All other features are considered to be Generally Available (GA). @@ -34,6 +34,14 @@ Characteristics of beta features: Your Support Contract provides **commercially-reasonable effort** support for Beta features, with the expectation that issues will require extra time and assistance from development to troubleshoot. +## Limited Availability (LA) + +Characteristics of limited availability features: + +- Ready for production use by a small set of customers. +- Can be booked by Deal Desk as part of an order. +- Fully documented and [supported](https://about.gitlab.com/support/statement-of-support/#starter-premium-and-ultimate-users). + ## Generally Available (GA) Generally Available features means that they passed the [Production Readiness Review](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/.gitlab/issue_templates/production_readiness.md) for GitLab.com, and are: diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index f05eaa677c1..8ff4e4c5035 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -123,6 +123,9 @@ the current stable release, and two previous monthly releases. In rare cases a r For instance, if we release `13.2.1` with a fix for a severe bug introduced in `13.0.0`, we could backport the fix to a new `13.0.x`, and `13.1.x` patch release. +Note that [severity](../development/contributing/issue_workflow.md#severity-labels) 3 and lower +requests will be automatically turned down. + To request backporting to more than one stable release for consideration, raise an issue in the [release/tasks](https://gitlab.com/gitlab-org/release/tasks/-/issues/new?issuable_template=Backporting-request) issue tracker. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 5458e9952fa..7ff03989c61 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -379,18 +379,24 @@ sudo -u git -H bundle exec rake gitlab:backup:create GITLAB_BACKUP_MAX_CONCURREN > - Introduced in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `incremental_repository_backup`. Disabled by default. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355945) in GitLab 14.10. +> - `PREVIOUS_BACKUP` option [introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4184) in GitLab 15.0. FLAG: On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `incremental_repository_backup`. On GitLab.com, this feature is not available. Incremental backups can be faster than full backups because they only pack changes since the last backup into the backup -bundle for each repository. There must be an existing backup to create an incremental backup from and this backup will be overwritten. You can use the `BACKUP=timestamp_of_backup` option to choose which backup will be used. +bundle for each repository. There must be an existing backup to create an incremental backup from: + +- In GitLab 14.9 and 14.10, use the `BACKUP=<timestamp_of_backup>` option to choose the backup to use. The chosen previous backup is overwritten. +- In GitLab 15.0 and later, use the `PREVIOUS_BACKUP=<timestamp_of_backup>` option to choose the backup to use. By default, a backup file is created + as documented in the [Backup timestamp](#backup-timestamp) section. You can override the `[TIMESTAMP]` portion of the filename by setting the + [`BACKUP` environment variable](#backup-filename). To create an incremental backup, run: ```shell -sudo gitlab-backup create INCREMENTAL=yes +sudo gitlab-backup create INCREMENTAL=yes PREVIOUS_BACKUP=<timestamp_of_backup> ``` Incremental backups can also be created from [an untarred backup](#skipping-tar-creation) by using `SKIP=tar`: @@ -399,6 +405,27 @@ Incremental backups can also be created from [an untarred backup](#skipping-tar- sudo gitlab-backup create INCREMENTAL=yes SKIP=tar ``` +#### Back up specific repository storages + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86896) in GitLab 15.0. + +When using [multiple repository storages](../administration/repository_storage_paths.md), +repositories from specific repository storages can be backed up separately +using the `REPOSITORIES_STORAGES` option. The option accepts a comma-separated list of +storage names. + +For example, for Omnibus GitLab installations: + +```shell +sudo gitlab-backup create REPOSITORIES_STORAGES=storage1,storage2 +``` + +For example, for installations from source: + +```shell +sudo -u git -H bundle exec rake gitlab:backup:create REPOSITORIES_STORAGES=storage1,storage2 +``` + #### Uploading backups to a remote (cloud) storage You can let the backup script upload (using the [Fog library](http://fog.io/)) @@ -981,7 +1008,7 @@ This procedure assumes that: First ensure your backup tar file is in the backup directory described in the `gitlab.rb` configuration `gitlab_rails['backup_path']`. The default is -`/var/opt/gitlab/backups`. It needs to be owned by the `git` user. +`/var/opt/gitlab/backups`. The backup file needs to be owned by the `git` user. ```shell sudo cp 11493107454_2018_04_25_10.6.4-ce_gitlab_backup.tar /var/opt/gitlab/backups/ @@ -1177,6 +1204,61 @@ project or group from there: A feature request to provide direct restore of individual projects or groups is being discussed in [issue #17517](https://gitlab.com/gitlab-org/gitlab/-/issues/17517). +### Restore options + +The command line tool GitLab provides to restore from backup can accept more +options. + +#### Excluding tasks on restore + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/19347) in GitLab 14.10. + +You can exclude specific tasks on restore by adding the environment variable `SKIP`, whose values are a comma-separated list of the following options: + +- `db` (database) +- `uploads` (attachments) +- `builds` (CI job output logs) +- `artifacts` (CI job artifacts) +- `lfs` (LFS objects) +- `terraform_state` (Terraform states) +- `registry` (Container Registry images) +- `pages` (Pages content) +- `repositories` (Git repositories data) +- `packages` (Packages) + +For Omnibus GitLab packages: + +```shell +sudo gitlab-backup restore BACKUP=timestamp_of_backup SKIP=db,uploads +``` + +For installations from source: + +```shell +sudo -u git -H bundle exec rake gitlab:backup:restore BACKUP=timestamp_of_backup SKIP=db,uploads RAILS_ENV=production +``` + +#### Restore specific repository storages + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86896) in GitLab 15.0. + +When using [multiple repository storages](../administration/repository_storage_paths.md), +repositories from specific repository storages can be restored separately +using the `REPOSITORIES_STORAGES` option. The option accepts a comma-separated list of +storage names. + +For example, for Omnibus GitLab installations: + +```shell +sudo gitlab-backup restore BACKUP=timestamp_of_backup REPOSITORIES_STORAGES=storage1,storage2 +``` + +For example, for installations from source: + +```shell +sudo -u git -H bundle exec rake gitlab:backup:restore BACKUP=timestamp_of_backup REPOSITORIES_STORAGES=storage1,storage2 +``` + ## Alternative backup strategies If your GitLab instance contains a lot of Git repository data, you may find the @@ -1365,6 +1447,7 @@ To prepare the new server: 1. Copy the [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079) from the old server to avoid man-in-the-middle attack warnings. + See [Manually replicate the primary site’s SSH host keys](../administration/geo/replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) for example steps. 1. [Install and configure GitLab](https://about.gitlab.com/install) except [incoming email](../administration/incoming_email.md): 1. Install GitLab. diff --git a/doc/security/asset_proxy.md b/doc/security/asset_proxy.md index e4849b1b658..1ccc9bfd9be 100644 --- a/doc/security/asset_proxy.md +++ b/doc/security/asset_proxy.md @@ -48,11 +48,11 @@ To install a Camo server as an asset proxy: | Attribute | Description | |:-------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| - | `asset_proxy_enabled` | Enable proxying of assets. If enabled, requires: `asset_proxy_url`). | + | `asset_proxy_enabled` | Enable proxying of assets. If enabled, requires: `asset_proxy_url`. | | `asset_proxy_secret_key` | Shared secret with the asset proxy server. | | `asset_proxy_url` | URL of the asset proxy server. | - | `asset_proxy_whitelist` | (Deprecated: Use `asset_proxy_allowlist` instead) Assets that match these domain(s) are NOT proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed. | - | `asset_proxy_allowlist` | Assets that match these domain(s) are NOT proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed. | + | `asset_proxy_whitelist` | (Deprecated: Use `asset_proxy_allowlist` instead) Assets that match these domains are NOT proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed. | + | `asset_proxy_allowlist` | Assets that match these domains are NOT proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed. | 1. Restart the server for the changes to take effect. Each time you change any values for the asset proxy, you need to restart the server. diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index cdf99e8377d..ac532ee491a 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -70,6 +70,26 @@ For configuration information, see ## Non-configurable limits +### Git operations using SSH + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78373) in GitLab 14.7. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79419) in GitLab 14.8. + +GitLab rate limits Git operations using SSH by user account and project. If a request from a user for a Git operation +on a project exceeds the rate limit, GitLab drops further connection requests from that user for the project. + +The rate limit applies at the Git command ([plumbing](https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain)) level. +Each command has a rate limit of 600 per minute. For example: + +- `git push` has a rate limit of 600 per minute. +- `git pull` has its own rate limit of 600 per minute. + +Because the same commands are shared by `git-upload-pack`, `git pull`, and `git clone`, they share a rate limit. + +The requests/minute threshold for this rate limit is not configurable. Self-managed customers can disable this +rate limit by [disabling the feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) +with `Feature.disable(:rate_limit_gitlab_shell)`. + ### Repository archives > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25750) in GitLab 12.9. diff --git a/doc/security/ssh_keys_restrictions.md b/doc/security/ssh_keys_restrictions.md index 03bc0207cf5..272d840ef13 100644 --- a/doc/security/ssh_keys_restrictions.md +++ b/doc/security/ssh_keys_restrictions.md @@ -24,7 +24,7 @@ the minimum key length for each technology: 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). 1. Expand the **Visibility and access controls** section: -  +  If a restriction is imposed on any key type, users cannot upload new SSH keys that don't meet the requirement. Any existing keys that don't meet it are disabled but not removed and users cannot diff --git a/doc/security/unlock_user.md b/doc/security/unlock_user.md index f2ad6696b9a..efe9c5784ad 100644 --- a/doc/security/unlock_user.md +++ b/doc/security/unlock_user.md @@ -10,7 +10,7 @@ type: howto Users are locked after ten failed sign-in attempts. These users remain locked: - For 10 minutes, after which time they are automatically unlocked. -- Until an admin unlocks them from the [Admin Area](../user/admin_area/index.md) or the command line in under 10 minutes. +- Until an administrator unlocks them from the [Admin Area](../user/admin_area/index.md) or the command line in under 10 minutes. ## Unlock a user from the Admin Area diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index 07b35ccebe8..c3d445103c4 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -49,7 +49,7 @@ This behavior can be overridden: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Network**. 1. Expand the **Outbound requests** section: -  +  1. Select **Allow requests to the local network from web hooks and services**. NOTE: diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index 17eafb7633e..aaa6447cb6c 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -65,7 +65,7 @@ the tiers are no longer mentioned in GitLab documentation: - Merge requests: - [Full code quality reports in the code quality tab](../user/project/merge_requests/code_quality.md#code-quality-reports) - [Merge request approvals](../user/project/merge_requests/approvals/index.md) - - [Multiple assignees](../user/project/merge_requests/getting_started.md#multiple-assignees) + - [Multiple assignees](../user/project/merge_requests/index.md#assign-multiple-users) - [Approval Rule information for Reviewers](../user/project/merge_requests/reviews/index.md#approval-rule-information-for-reviewers) - [Required Approvals](../user/project/merge_requests/approvals/index.md#required-approvals) - [Code Owners as eligible approvers](../user/project/merge_requests/approvals/rules.md#code-owners-as-eligible-approvers) diff --git a/doc/subscriptions/gitlab_dedicated/index.md b/doc/subscriptions/gitlab_dedicated/index.md new file mode 100644 index 00000000000..7c768d1b403 --- /dev/null +++ b/doc/subscriptions/gitlab_dedicated/index.md @@ -0,0 +1,79 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# GitLab Dedicated + +NOTE: +GitLab Dedicated is currently in limited availability. Please [contact us](#contact-us) if you are interested. + +GitLab Dedicated is a fully isolated, single-tenant SaaS service that is: + +- Hosted and managed by GitLab, Inc. +- Deployed in a region of choice on AWS. + +GitLab Dedicated enables you to offload the operational overhead of managing the DevOps Platform. It offers a high level of tenant isolation and deployment customization, ideal for enterprises in highly-regulated industries. By deploying your GitLab instance onto separate Cloud Infrastructure from other tenants, GitLab Dedicated helps you better meet your security and compliance requirements. + +## Available features + +- Authentication: Support for instance-level [SAML OmniAuth](../../integration/saml.md) functionality. GitLab Dedicated acts as the service provider, and you will need to provide the necessary [configuration](../../integration/saml.md#general-setup) in order for GitLab to communicate with your IdP. This will be provided during onboarding. SAML [request signing](../../integration/saml.md#request-signing-optional) is supported. +- Networking: + - Public connectivity + - Optional. Private connectivity via [AWS PrivateLink](https://aws.amazon.com/privatelink/). + You can specify an AWS IAM Principal and preferred Availability Zones during onboarding to enable this functionality. +- Upgrade strategy: + - Monthly upgrades tracking one release behind the latest (n-1), with the latest security release. + - Out of band security patches provided for high severity releases. +- Backup strategy: regular backups taken and tested. +- Choice of cloud region: upon onboarding, choose the cloud region where you want to deploy your instance. Some AWS regions have limited features and as a result, we are not able to deploy production instances to those regions. See below for the [full list of regions](#aws-regions-not-supported) not currently supported. +- Security: Data encrypted at rest and in transit using latest encryption standards. +- Application: Self-managed [Ultimate feature set](https://about.gitlab.com/pricing/self-managed/feature-comparison/) with the exception of the unsupported features [listed below](#features-not-available-at-launch). + +## Features not available at launch + +Features that are not available but we plan to support in the future: + +- LDAP, Smartcard, Kerberos authentication +- Custom domain +- Advanced Search +- Pages +- GitLab-managed runners +- FortiAuthenticator/FortiToken 2FA +- Reply-by email +- Service desk + +Features that we do not plan to offer at all: + +- Mattermost +- Server-side Git Hooks + +### AWS regions not supported + +The following AWS regions are not available at launch: + +- Jakarta (ap-southeast-3) +- Bahrain (me-south-1) +- Hong Kong (ap-east-1) +- Cape Town (af-south-1) +- Milan (eu-south-1) +- Paris (eu-west-3) +- GovCloud + +## Contact us + +Fill in the following form to contact us and learn more about this offering. + +<!-- markdownlint-disable --> + +<!-- NOTE: The following form only shows when the site is served under HTTPS, + so it will not appear when developing locally or in a review app. + See https://gitlab.com/gitlab-com/marketing/marketing-operations/-/issues/6238#note_923358643 +--> + +<script src="//page.gitlab.com/js/forms2/js/forms2.min.js"></script> +<form id="mktoForm_3226"></form> +<script>MktoForms2.loadForm("//page.gitlab.com", "194-VVC-221", 3226);</script> + +<!-- markdownlint-enable --> diff --git a/doc/subscriptions/img/support_diagram_c.png b/doc/subscriptions/img/support_diagram_c.png Binary files differindex a2fed80e912..e98baed8605 100644 --- a/doc/subscriptions/img/support_diagram_c.png +++ b/doc/subscriptions/img/support_diagram_c.png diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 824fe0bacde..58af5787f2b 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -30,6 +30,7 @@ GitLab SaaS or GitLab self-managed: - [GitLab SaaS](gitlab_com/index.md): The GitLab software-as-a-service offering. You don't need to install anything to use GitLab SaaS, you only need to [sign up](https://gitlab.com/users/sign_up) and start using GitLab straight away. +- [GitLab Dedicated](gitlab_dedicated/index.md): a single-tenant SaaS service for highly regulated and large enterprises. - [GitLab self-managed](self_managed/index.md): Install, administer, and maintain your own GitLab instance. diff --git a/doc/subscriptions/quarterly_reconciliation.md b/doc/subscriptions/quarterly_reconciliation.md index 3398902da1b..78c844b897c 100644 --- a/doc/subscriptions/quarterly_reconciliation.md +++ b/doc/subscriptions/quarterly_reconciliation.md @@ -70,7 +70,7 @@ sent and subject to your terms. ### Self-managed instances -Admins receive an email **six days after the reconciliation date**. +Administrators receive an email **six days after the reconciliation date**. This email communicates the [overage seat quantity](self_managed/index.md#users-over-license) and expected invoice amount. diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index 6765ca19518..ce49b5a9c05 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -425,7 +425,7 @@ an expiration message is displayed to all administrators. For GitLab self-managed instances, you have a 14-day grace period before this occurs. -- To resume functionality, acticate a new license. +- To resume functionality, activate a new license. - To fall back to Free features, delete the expired license. ## Contact Support diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index 503774ef6b5..e2d984dbbff 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -26,7 +26,7 @@ used for the build. Specify either: -- The CI/CD variable `BUILDPACK_URL` according to [`pack`'s specifications](https://buildpacks.io/docs/app-developer-guide/specify-buildpacks/). +- The CI/CD variable `BUILDPACK_URL` with any of [`pack`'s URI specification formats](https://buildpacks.io/docs/app-developer-guide/specify-buildpacks/). - A [`project.toml` project descriptor](https://buildpacks.io/docs/app-developer-guide/using-project-descriptor/) with the buildpacks you would like to include. ### Custom buildpacks with Herokuish @@ -431,7 +431,7 @@ applications. | `HELM_UPGRADE_EXTRA_ARGS` | Allows extra options in `helm upgrade` commands when deploying the application. Note that using quotes doesn't prevent word splitting. | | `INCREMENTAL_ROLLOUT_MODE` | If present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. | | `K8S_SECRET_*` | Any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) is made available by Auto DevOps as environment variables to the deployed application. | -| `KUBE_CONTEXT` | From GitLab 14.5, can be used to select a context to use from `KUBECONFIG`. When `KUBE_CONTEXT` is blank, the default context in `KUBECONFIG` (if any) is used. A context must be selected when used [with the agent for Kubernetes](../../user/clusters/agent/ci_cd_tunnel.md). | +| `KUBE_CONTEXT` | From GitLab 14.5, can be used to select a context to use from `KUBECONFIG`. When `KUBE_CONTEXT` is blank, the default context in `KUBECONFIG` (if any) is used. A context must be selected when used [with the agent for Kubernetes](../../user/clusters/agent/ci_cd_workflow.md). | | `KUBE_INGRESS_BASE_DOMAIN` | Can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/gitlab_managed_clusters.md#base-domain) for more information. | | `KUBE_NAMESPACE` | The namespace used for deployments. When using certificate-based clusters, [this value should not be overwritten directly](../../user/project/clusters/deploy_to_cluster.md#custom-namespace). | | `KUBECONFIG` | The kubeconfig to use for deployments. User-provided values take priority over GitLab-provided values. | @@ -479,7 +479,6 @@ The following table lists variables used to disable jobs. | `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | | `bandit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | | `brakeman-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `bundler-audit-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `canary` | `CANARY_ENABLED` | | This manual job is created if the variable is present. | | `cluster_image_scanning` | `CLUSTER_IMAGE_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `code_intelligence` | `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6 | If the variable is present, the job isn't created. | @@ -503,7 +502,6 @@ The following table lists variables used to disable jobs. | `browser_performance` | `BROWSER_PERFORMANCE_DISABLED` | From GitLab 14.0 | Browser performance. If the variable is present, the job isn't created. Replaces `performance`. | | `phpcs-security-audit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | | `pmd-apex-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `retire-js-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | | `review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | | `review:stop` | `REVIEW_DISABLED` | | Manual job. If the variable is present, the job isn't created. | | `sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | diff --git a/doc/topics/autodevops/prepare_deployment.md b/doc/topics/autodevops/prepare_deployment.md index c23774b1ffd..7c9bf5a770e 100644 --- a/doc/topics/autodevops/prepare_deployment.md +++ b/doc/topics/autodevops/prepare_deployment.md @@ -51,7 +51,7 @@ as other environment [variables](../../ci/variables/index.md#cicd-variable-prece If you don't specify the base domain in your projects and groups, Auto DevOps uses the instance-wide **Auto DevOps domain**. -Auto DevOps requires a wildcard DNS A record matching the base domain(s). For +Auto DevOps requires a wildcard DNS A record matching the base domains. For a base domain of `example.com`, you'd need a DNS entry like: ```plaintext diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 13d831aa00d..8d1bf7adc7f 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -296,7 +296,8 @@ bin/rails test test/controllers/welcome_controller_test.rb:4 To fix the broken test: -1. Return to the **Overview** page for your merge request, and select **Open in Web IDE**. +1. Return to your merge request. +1. In the upper right corner, select **Code**, then select **Open in Gitpod**. 1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` file, and select it to open it. 1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md index 816cbbece4f..039d98efd47 100644 --- a/doc/topics/autodevops/requirements.md +++ b/doc/topics/autodevops/requirements.md @@ -69,7 +69,7 @@ as other environment [variables](../../ci/variables/index.md#cicd-variable-prece If you don't specify the base domain in your projects and groups, Auto DevOps uses the instance-wide **Auto DevOps domain**. -Auto DevOps requires a wildcard DNS A record matching the base domain(s). For +Auto DevOps requires a wildcard DNS `A` record that matches the base domains. For a base domain of `example.com`, you'd need a DNS entry like: ```plaintext diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 790b46b6310..a677787b980 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -50,7 +50,7 @@ the CI/CD variable `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER`. Each buildpack requires your project's repository to contain certain files for Auto Build to build your application successfully. The structure is specific to the builder and buildpacks you have selected. -For example, when using the Heroku's builder (the default), your application's +For example, when using the Heroku builder (the default), your application's root directory must contain the appropriate file for your application's language: @@ -240,7 +240,7 @@ To learn more about [License Compliance](../../user/compliance/license_compliance/index.md), see the documentation. -## Auto Container Scanning **(ULTIMATE)** +## Auto Container Scanning Vulnerability static analysis for containers uses [Trivy](https://aquasecurity.github.io/trivy/latest/) to check for potential security issues in Docker images. The Auto Container Scanning stage is @@ -551,129 +551,6 @@ workers: terminationGracePeriodSeconds: 60 ``` -### Network Policy - -- [Introduced](https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/merge_requests/30) in GitLab 12.7. -- [Deprecated](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/184) in GitLab 13.9. - -By default, all Kubernetes pods are -[non-isolated](https://kubernetes.io/docs/concepts/services-networking/network-policies/#isolated-and-non-isolated-pods), -and accept traffic to and from any source. You can use -[NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -to restrict connections to and from selected pods, namespaces, and the Internet. - -NOTE: -You must use a Kubernetes network plugin that implements support for -`NetworkPolicy`. The default network plugin for Kubernetes (`kubenet`) -[does not implement](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#kubenet) -support for it. The [Cilium](https://cilium.io/) network plugin can be -installed as a [cluster application](../../user/project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium) -to enable support for network policies. - -You can enable deployment of a network policy by setting the following -in the `.gitlab/auto-deploy-values.yaml` file: - -```yaml -networkPolicy: - enabled: true -``` - -The default policy deployed by the Auto Deploy pipeline allows -traffic within a local namespace, and from the `gitlab-managed-apps` -namespace. All other inbound connections are blocked. Outbound -traffic (for example, to the Internet) is not affected by the default policy. - -You can also provide a custom [policy specification](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -in the `.gitlab/auto-deploy-values.yaml` file, for example: - -```yaml -networkPolicy: - enabled: true - spec: - podSelector: - matchLabels: - app.gitlab.com/env: staging - ingress: - - from: - - podSelector: - matchLabels: {} - - namespaceSelector: - matchLabels: - app.gitlab.com/managed_by: gitlab -``` - -For more information on installing Network Policies, see -[Use the Cluster Management Template to Install Cilium](../../user/project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). - -### Cilium Network Policy - -> [Introduced](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/merge_requests/184) in GitLab 13.9. - -By default, all Kubernetes pods are -[non-isolated](https://kubernetes.io/docs/concepts/services-networking/network-policies/#isolated-and-non-isolated-pods), -and accept traffic to and from any source. You can use -[CiliumNetworkPolicy](https://docs.cilium.io/en/v1.8/concepts/kubernetes/policy/#ciliumnetworkpolicy) -to restrict connections to and from selected pods, namespaces, and the internet. - -#### Requirements - -As the default network plugin for Kubernetes (`kubenet`) -[does not implement](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#kubenet) -support for it, you must have [Cilium](https://docs.cilium.io/en/v1.8/intro/) as your Kubernetes network plugin. - -The [Cilium](https://cilium.io/) network plugin can be -installed with a [cluster management project template](../../user/project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium) -to enable support for network policies. - -#### Configuration - -You can enable deployment of a network policy by setting the following -in the `.gitlab/auto-deploy-values.yaml` file: - -```yaml -ciliumNetworkPolicy: - enabled: true -``` - -The default policy deployed by the Auto Deploy pipeline allows -traffic within a local namespace, and from the `gitlab-managed-apps` -namespace. All other inbound connections are blocked. Outbound -traffic (for example, to the internet) is not affected by the default policy. - -You can also provide a custom [policy specification](https://docs.cilium.io/en/v1.8/policy/language/#simple-ingress-allow) -in the `.gitlab/auto-deploy-values.yaml` file, for example: - -```yaml -ciliumNetworkPolicy: - enabled: true - spec: - endpointSelector: - matchLabels: - app.gitlab.com/env: staging - ingress: - - fromEndpoints: - - matchLabels: - app.gitlab.com/managed_by: gitlab -``` - -#### Enabling Alerts - -You can also enable alerts. Network policies with alerts are considered only if -the [agent](../../user/clusters/agent/index.md) -has been integrated. - -You can enable alerts as follows: - -```yaml -ciliumNetworkPolicy: - enabled: true - alerts: - enabled: true -``` - -For more information on installing Network Policies, see -[Use the Cluster Management Template to Install Cilium](../../user/project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). - ### Running commands in the container Applications built with [Auto Build](#auto-build) using Herokuish, the default diff --git a/doc/topics/git/feature_branching.md b/doc/topics/git/feature_branching.md index f0ded5511ee..d3b2510f4e8 100644 --- a/doc/topics/git/feature_branching.md +++ b/doc/topics/git/feature_branching.md @@ -12,7 +12,7 @@ comments: false - Keeps changes isolated - Consider a 1-to-1 link to issues - Push branches to the server frequently - - Hint: This is a cheap backup for your work-in-progress code + - Hint: Pushing branches is a cheap backup for your work-in-progress code. ## Feature branching sample workflow diff --git a/doc/topics/git/git_rebase.md b/doc/topics/git/git_rebase.md index c0bc7ed4e5c..87fce0b29ff 100644 --- a/doc/topics/git/git_rebase.md +++ b/doc/topics/git/git_rebase.md @@ -17,7 +17,7 @@ Before diving into this document, make sure you are familiar with using ## Git rebase [Rebasing](https://git-scm.com/docs/git-rebase) is a very common operation in -Git. There are the following rebase options: +Git, and has these options: - [Regular rebase](#regular-rebase). - [Interactive rebase](#interactive-rebase). @@ -69,15 +69,15 @@ changes by resetting `my-feature-branch` against `my-feature-branch-backup`: git reset --hard my-feature-branch-backup ``` -Note that if you added changes to `my-feature-branch` after creating the backup branch, +If you added changes to `my-feature-branch` after creating the backup branch, you lose them when resetting. ### Regular rebase With a regular rebase you can update your feature branch with the default branch (or any other branch). -This is an important step for Git-based development strategies. You can -ensure that the changes you're adding to the codebase do not break any +This step is important for Git-based development strategies. You can +ensure your new changes don't break any existing changes added to the target branch _after_ you created your feature branch. @@ -148,7 +148,7 @@ executes it as soon as possible. The user performing the rebase action is considered a user that added commits to the merge request. When the merge request approvals setting [**Prevent approvals by users who add commits**](../../user/project/merge_requests/approvals/settings.md#prevent-approvals-by-users-who-add-commits) -is enabled, this setting prevents the user from also approving the merge request. +is enabled, the user can't also approve the merge request. ### Interactive rebase @@ -158,7 +158,7 @@ commits. Use a rebase for changing past commit messages, and organizing the commit history of your branch to keep it clean. NOTE: -If you want to keep the default branch commit history clean, you don't need to +Keeping the default branch commit history clean doesn't require you to manually squash all your commits before merging every merge request. With [Squash and Merge](../../user/project/merge_requests/squash_and_merge.md), GitLab does it automatically. @@ -176,18 +176,17 @@ git rebase -i HEAD~3 Git opens the last three commits in your terminal text editor and describes all the interactive rebase options you can use. The default option is `pick`, which maintains the commit unchanged. Replace the keyword `pick` according to -the operation you want to perform in each commit. To do so, you need to edit +the operation you want to perform in each commit. To do so, edit the commits in your terminal's text editor. -For example, if you're using [Vim](https://www.vim.org/) as the text editor in -a macOS's `ZSH` shell, and you want to `squash` or `fixup` all the three commits -(join them into one): +For example, with [Vim](https://www.vim.org/) as the text editor in +a macOS's `ZSH` shell, you can `squash` or `fixup` (combine) all three commits: <!-- vale gitlab.FirstPerson = NO --> 1. Press <kbd>i</kbd> on your keyboard to switch to Vim's editing mode. -1. Navigate with your keyboard arrows to edit the **second** commit keyword +1. Use your keyboard arrows to edit the **second** commit keyword from `pick` to `squash` or `fixup` (or `s` or `f`). Do the same to the **third** commit. The first commit should be left **unchanged** (`pick`) as we want to squash the second and third into the first. @@ -204,7 +203,7 @@ a macOS's `ZSH` shell, and you want to `squash` or `fixup` all the three commits <!-- vale gitlab.FirstPerson = YES --> -Note that the steps for editing through the command line can be slightly +The steps for editing through the command line can be slightly different depending on your operating system and the shell you're using. See [Numerous undo possibilities in Git](numerous_undo_possibilities_in_git/index.md#undo-staged-local-changes-without-modifying-history) @@ -226,8 +225,8 @@ Forcing an update is **not** recommended when you're working on shared branches. Alternatively, you can pass the flag [`--force-with-lease`](https://git-scm.com/docs/git-push#Documentation/git-push.txt---force-with-leaseltrefnamegt) -instead. It is safer, as it does not overwrite any work on the remote -branch if more commits were added to the remote branch by someone else: +instead, as it is safer. This flag preserves any new commits added to the remote +branch by other people: ```shell git push --force-with-lease origin my-feature-branch diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index db63cee3523..410d2150de5 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -45,7 +45,7 @@ Documentation for GitLab instance administrators is under [LFS administration do ## Using Git LFS -Let's take a look at the workflow when you need to check large files into your Git +Let's take a look at the workflow for checking large files into your Git repository with Git LFS. For example, if you want to upload a very large file and check it into your Git repository: @@ -130,10 +130,10 @@ Technical details about how this works can be found in the [development document ## Troubleshooting -### Encountered `n` file(s) that should have been pointers, but weren't +### Encountered `n` files that should have been pointers, but weren't -This error indicates the file (or files) are expected to be tracked by LFS, but for -some reason the repository is not tracking them as LFS. This issue can be one +This error indicates the files are expected to be tracked by LFS, but +the repository is not tracking them as LFS. This issue can be one potential reason for this error: [Files not tracked with LFS when uploaded through the web interface](https://gitlab.com/gitlab-org/gitlab/-/issues/326342#note_586820485) @@ -160,7 +160,7 @@ To resolve the problem, migrate the affected file (or files) and push back to th ### error: Repository or object not found -There are a couple of reasons why this error can occur: +This error can occur for a few reasons, including: - You don't have permissions to access certain LFS object @@ -232,7 +232,7 @@ Git LFS authenticates the user with HTTP Basic Authentication on every push for every object, so user HTTPS credentials are required. By default, Git has support for remembering the credentials for each repository -you use. This is described in [Git credentials man pages](https://git-scm.com/docs/gitcredentials). +you use. To learn more, read the [Git credentials man pages](https://git-scm.com/docs/gitcredentials). For example, you can tell Git to remember the password for a period of time in which you expect to push the objects: diff --git a/doc/topics/git/merge_conflicts.md b/doc/topics/git/merge_conflicts.md index 47276ccb0b2..03b7c03c02a 100644 --- a/doc/topics/git/merge_conflicts.md +++ b/doc/topics/git/merge_conflicts.md @@ -14,12 +14,12 @@ comments: false ## Merge conflicts sample workflow -1. Checkout a new branch and edit `conflicts.rb`. Add 'Line4' and 'Line5'. +1. Check out a new branch and edit `conflicts.rb`. Add 'Line4' and 'Line5'. 1. Commit and push. -1. Checkout master and edit `conflicts.rb`. Add 'Line6' and 'Line7' below 'Line3'. -1. Commit and push to master. +1. Check out `main` and edit `conflicts.rb`. Add 'Line6' and 'Line7' below 'Line3'. +1. Commit and push to `main``. 1. Create a merge request and watch it fail. -1. Rebase our new branch with master. +1. Rebase our new branch with `main`. 1. Fix conflicts on the `conflicts.rb` file. 1. Stage the file and continue rebasing. 1. Force push the changes. @@ -34,12 +34,12 @@ git checkout -b conflicts_branch git commit -am "add line4 and line5" git push origin conflicts_branch -git checkout master +git checkout main # vi conflicts.rb # Add 'Line6' and 'Line7' git commit -am "add line6 and line7" -git push origin master +git push origin main ``` Create a merge request on the GitLab web UI, and a conflict warning displays. @@ -47,7 +47,7 @@ Create a merge request on the GitLab web UI, and a conflict warning displays. ```shell git checkout conflicts_branch git fetch -git rebase master +git rebase main # Fix conflicts by editing the files. @@ -64,6 +64,6 @@ git push origin conflicts_branch -f ## Note - When to use `git merge` and when to use `git rebase` -- Rebase when updating your branch with master -- Merge when bringing changes from feature to master +- Rebase when updating your branch with `main` +- Merge when bringing changes from feature to `main` - Reference: <https://www.atlassian.com/git/tutorials/merging-vs-rebasing> diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index cad29d30af4..91ff4d69c2f 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -26,7 +26,7 @@ Git 2.22.0 or later is required. Storing large binary files in Git is normally discouraged, because every large file added is downloaded by everyone who clones or fetches changes -thereafter. This is slow, if not a complete obstruction when working from a slow +thereafter. These downloads are slow and problematic, especially when working from a slow or unreliable internet connection. Using partial clone with a file size filter solves this problem, by excluding @@ -65,7 +65,7 @@ The output is longer because Git: 1. Clones the repository excluding files larger than 1 megabyte. 1. Downloads any missing large files needed to check out the default branch. -When changing branches, Git may need to download more missing files. +When changing branches, Git may download more missing files. ## Filter by object type @@ -147,7 +147,7 @@ For more details, see the Git documentation for ``` 1. Clone and filter by path. Support for `--filter=sparse:oid` using the - clone command is not yet fully integrated with sparse checkout. + clone command is not fully integrated with sparse checkout. ```shell @@ -162,8 +162,7 @@ For more details, see the Git documentation for WARNING: Git integrations with `bash`, `zsh`, etc and editors that automatically show Git status information often run `git fetch` which fetches the - entire repository. You many need to disable or reconfigure these - integrations. + entire repository. Disabling or reconfiguring these integrations might be required. ## Remove partial clone filtering diff --git a/doc/topics/git/stash.md b/doc/topics/git/stash.md index d321795e034..638ab559bd3 100644 --- a/doc/topics/git/stash.md +++ b/doc/topics/git/stash.md @@ -7,8 +7,8 @@ comments: false # Git Stash **(FREE)** -We use `git stash` to store our changes when they are not ready to be committed -and we need to change to a different branch. +We use `git stash` to store our changes when they are not ready to be committed, +but we must change to a different branch. - Stash: @@ -37,7 +37,7 @@ and we need to change to a different branch. git stash list --stat ``` -- To clean our stack we need to manually remove them: +- To clean our stack, manually remove them: ```shell # drop top stash @@ -54,7 +54,7 @@ and we need to change to a different branch. git stash pop ``` -- If we meet conflicts we need to either reset or commit our changes. +- If we meet conflicts, either reset or commit our changes. - Conflicts through `pop` doesn't drop a stash afterwards. ## Git Stash sample workflow diff --git a/doc/topics/git/tags.md b/doc/topics/git/tags.md index 8576bcd09ed..d3237fda968 100644 --- a/doc/topics/git/tags.md +++ b/doc/topics/git/tags.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Tags **(FREE)** -Tags are useful for marking certain deployments and releases for later +Tags help you mark certain deployments and releases for later reference. Git supports two types of tags: - Annotated tags: An unchangeable part of Git history. diff --git a/doc/topics/git/terminology.md b/doc/topics/git/terminology.md index 35814543934..4ce87aa2d11 100644 --- a/doc/topics/git/terminology.md +++ b/doc/topics/git/terminology.md @@ -30,7 +30,7 @@ When you want to contribute to someone else's repository, you make a copy of it. This copy is called a [**fork**](../../user/project/repository/forking_workflow.md#creating-a-fork). The process is called "creating a fork." -When you fork a repo, you create a copy of the project in your own +When you fork a repository, you create a copy of the project in your own [namespace](../../user/group/#namespaces). You then have write permissions to modify the project files and settings. @@ -53,10 +53,10 @@ upload the changes to the remote repository on GitLab. ## Pull and push After you save a local copy of a repository and modify the files on your computer, you can upload the -changes to GitLab. This is referred to as **pushing** to the remote, because you use the command +changes to GitLab. This action is known as **pushing** to the remote, because you use the command [`git push`](../../gitlab-basics/start-using-git.md#send-changes-to-gitlabcom). When the remote repository changes, your local copy is behind. You can update your local copy with the new changes in the remote repository. -This is referred to as **pulling** from the remote, because you use the command +This action is known as **pulling** from the remote, because you use the command [`git pull`](../../gitlab-basics/start-using-git.md#download-the-latest-changes-in-the-project). diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index 0aadde7f7c2..13962ad0376 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -214,7 +214,7 @@ apply more than one: ```shell omnibus_gitconfig['system'] = { # Set the http.postBuffer size, in bytes - "http" => ["postBuffer => 524288000"] + "http" => ["postBuffer = 524288000"] } ``` diff --git a/doc/topics/git/useful_git_commands.md b/doc/topics/git/useful_git_commands.md index 61f170d934a..13a40dd58ca 100644 --- a/doc/topics/git/useful_git_commands.md +++ b/doc/topics/git/useful_git_commands.md @@ -8,7 +8,7 @@ type: reference # Useful Git commands **(FREE)** The GitLab support team has collected these commands to help you. You may not -need to use them often. +need them frequently. ## Remotes @@ -42,7 +42,7 @@ git reset <filename> ### Revert a file to HEAD state and remove changes -There are two options to revert changes to a file: +To revert changes to a file, you can use either: - `git checkout <filename>` - `git reset --hard <filename>` diff --git a/doc/topics/release_your_application.md b/doc/topics/release_your_application.md index c791b1f7185..6c94e9e78f9 100644 --- a/doc/topics/release_your_application.md +++ b/doc/topics/release_your_application.md @@ -37,7 +37,7 @@ approach to manage Kubernetes deployments. #### Deploy to Kubernetes from GitLab CI/CD With the [GitLab agent for Kubernetes](../user/clusters/agent/install/index.md), you can perform [push-based -deployments](../user/clusters/agent/ci_cd_tunnel.md) from GitLab CI/CD. The agent provides +deployments](../user/clusters/agent/ci_cd_workflow.md) from GitLab CI/CD. The agent provides a secure and reliable connection between GitLab and your Kubernetes cluster. ### Deploy to AWS with GitLab CI/CD @@ -67,5 +67,5 @@ Use [feature flags](../operations/feature_flags.md) to control and strategically ## Deploy to Google Cloud -GitLab [Cloud Seed](../cloud_seed/index.md) is an open-source Incubation Engineering program that +GitLab [Cloud Seed](../cloud_seed/index.md) is an open-source Incubation Engineering program that enables you to set up deployment credentials and deploy your application to Google Cloud Run with minimal friction. diff --git a/doc/tutorials/img/branches_dropdown_v14_10.png b/doc/tutorials/img/branches_dropdown_v14_10.png Binary files differindex 73d94f9823a..926baff0ff8 100644 --- a/doc/tutorials/img/branches_dropdown_v14_10.png +++ b/doc/tutorials/img/branches_dropdown_v14_10.png diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index cf3c23a99a7..55f8321b559 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -20,7 +20,7 @@ and running quickly. | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Use GitLab for DevOps](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) (12m 34s) | Use GitLab through the entire DevOps lifecycle, from planning to monitoring. | **{star}** | | [Use Markdown at GitLab](../user/markdown.md) | GitLab Flavored Markdown (GLFM) is used in many areas of GitLab, for example, in merge requests. | **{star}** | | [GitLab 201](https://gitlab.edcast.com/pathways/ECL-44010cf6-7a9c-4b9b-b684-fa08508a3252) | Go beyond the basics to learn more about using GitLab for your work. | | -| Learn GitLab project | You might already have the **Learn GitLab** project, which has tutorial-style issues to help you learn GitLab. If not, download [this export file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/project_templates/learn_gitlab_ultimate.tar.gz) and [import it to a new project](../user/project/settings/import_export.md#import-a-project-and-its-data). | | +| <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Learn GitLab project walkthrough](https://www.youtube.com/watch?v=-oaI2WEKdI4&list=PL05JrBw4t0KofkHq4GZJ05FnNGa11PQ4d) (59m 2s) | Step through the tutorial-style issues in the **Learn GitLab** project. If you don't have this project, download [the export file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/project_templates/learn_gitlab_ultimate.tar.gz) and [import it to a new project](../user/project/settings/import_export.md#import-a-project-and-its-data). | | | [Productivity tips](https://about.gitlab.com/blog/2021/02/18/improve-your-gitlab-productivity-with-these-10-tips/) | Get tips to help make you a productive GitLab user. | | | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Structure a multi-team organization](https://www.youtube.com/watch?v=KmASFwSap7c) (37m 37s) | Learn to use issues, milestones, epics, labels, and more to plan and manage your work. | | @@ -66,7 +66,7 @@ configure the infrastructure for your application. | Topic | Description | Good for beginners | |-------|-------------|--------------------| -| [Connect with a Kubernetes cluster](https://about.gitlab.com/blog/2021/11/18/gitops-with-gitlab-connecting-the-cluster/) | Connect a Kubernetes cluster with GitLab for pull and push based deployments and security integrations. | | +| [Use GitOps with GitLab](https://about.gitlab.com/blog/2022/04/07/the-ultimate-guide-to-gitops-with-gitlab/) | Learn how to provision and manage a base infrastructure, connect to a Kubernetes cluster, deploy third-party applications, and carry out other infrastructure automation tasks. | | | [Use Auto DevOps to deploy an application](../topics/autodevops/quick_start_guide.md) | Deploy an application to Google Kubernetes Engine (GKE). | | ## Publish a static website diff --git a/doc/tutorials/make_your_first_git_commit.md b/doc/tutorials/make_your_first_git_commit.md index 4b88b528be6..fd153b51a2c 100644 --- a/doc/tutorials/make_your_first_git_commit.md +++ b/doc/tutorials/make_your_first_git_commit.md @@ -255,8 +255,6 @@ Let's look in the UI and confirm your changes. Go to your project.  -- Above the file list, select **History** to view your commit details. - Now you can return to the command line and change back to your personal branch (`git checkout example-tutorial-branch`). You can continue updating files or creating new ones. Type `git status` to view the status diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 7e8eaef2b4c..903ae9c714c 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -36,14 +36,112 @@ For deprecation reviewers (Technical Writers only): https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-doc --> +{::options parse_block_html="true" /} + View deprecations by the product versions in which they were announced. Each deprecation has a **planned removal milestone** and indicates whether it is a breaking change. Most of the deprecations are **planned for removal in 15.0**, and many of them are **breaking changes**. +<div class="js-deprecation-filters"></div> + +<div class="announcement-milestone"> + +## 15.0 + +<div class="deprecation removal-160 breaking-change"> + +### CiCdSettingsUpdate mutation renamed to ProjectCiCdSettingsUpdate + +WARNING: +This feature will be changed or removed in 16.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The `CiCdSettingsUpdate` mutation was renamed to `ProjectCiCdSettingsUpdate` in GitLab 15.0. +The `CiCdSettingsUpdate` mutation will be removed in GitLab 16.0. +Any user scripts that use the `CiCdSettingsUpdate` mutation must be updated to use `ProjectCiCdSettingsUpdate` +instead. + +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> + +### GraphQL API legacyMode argument for Runner status + +WARNING: +This feature will be changed or removed in 16.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The `legacyMode` argument to the `status` field in `RunnerType` will be rendered non-functional in the 16.0 release +as part of the deprecations details in the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351109). + +In GitLab 16.0 and later, the `status` field will act as if `legacyMode` is null. The `legacyMode` argument will +be present during the 16.x cycle to avoid breaking the API signature, and will be removed altogether in the +17.0 release. + +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> + +### PostgreSQL 12 deprecated + +WARNING: +This feature will be changed or removed in 16.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Support for PostgreSQL 12 is scheduled for removal in GitLab 16.0. +In GitLab 16.0, PostgreSQL 13 becomes the minimum required PostgreSQL version. + +PostgreSQL 12 will be supported for the full GitLab 15 release cycle. +PostgreSQL 13 will also be supported for instances that want to upgrade prior to GitLab 16.0. + +Upgrading to PostgreSQL 13 is not yet supported for GitLab instances with Geo enabled. Geo support for PostgreSQL 13 will be announced in a minor release version of GitLab 15, after the process is fully supported and validated. For more information, read the Geo related verifications on the [support epic for PostgreSQL 13](https://gitlab.com/groups/gitlab-org/-/epics/3832). + +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-152"> + +### Vulnerability Report sort by State + +The ability to sort the Vulnerability Report by the `State` column was disabled and put behind a feature flag in GitLab 14.10 due to a refactor +of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting +by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed in +GitLab 15.2 to simplify the codebase and prevent any unwanted performance degradation. + +**Planned removal milestone: <span class="removal-milestone">15.2</span> (2022-07-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.10 +<div class="deprecation removal-150 breaking-change"> + +### Dependency Scanning default Java version changed to 17 + +WARNING: +This feature will be changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GitLab 15.0, for Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_Java_Version` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). + +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> + ### Manual iteration management WARNING: @@ -72,7 +170,10 @@ arguments will be removed: For more information about iteration cadences, you can refer to [the documentation of the feature](https://docs.gitlab.com/ee/user/group/iterations/#iteration-cadences). -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Outdated indices of Advanced Search migrations @@ -84,7 +185,10 @@ changes to your code, settings, or workflow. As Advanced Search migrations usually require support multiple code paths for a long period of time, it’s important to clean those up when we safely can. We use GitLab major version upgrades as a safe time to remove backward compatibility for indices that have not been fully migrated. See the [upgrade documentation](https://docs.gitlab.com/ee/update/index.html#upgrading-to-a-new-major-version) for details. -**Planned removal milestone: 15.0 (2021-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### Toggle notes confidentiality on APIs @@ -96,10 +200,16 @@ changes to your code, settings, or workflow. Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. Updating notes confidential attribute is no longer supported by any means. We are changing this to simplify the experience and prevent private information from being unintentionally exposed. -**Planned removal milestone: 16.0 (2023-05-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.9 +<div class="deprecation removal-150 breaking-change"> + ### Background upload for object storage WARNING: @@ -117,19 +227,28 @@ This impacts a small subset of object storage providers: GitLab will publish additional guidance to assist affected customers in migrating. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-151"> ### Deprecate support for Debian 9 Long term service and support (LTSS) for [Debian 9 Stretch ends in July 2022](https://wiki.debian.org/LTS). Therefore, we will no longer support the Debian 9 distribution for the GitLab package. Users can upgrade to Debian 10 or Debian 11. -**Planned removal milestone: 15.1 (2022-06-22)** +**Planned removal milestone: <span class="removal-milestone">15.1</span> (2022-06-22)** +</div> + +<div class="deprecation removal-150"> ### GitLab Pages running as daemon In 15.0, support for daemon mode for GitLab Pages will be removed. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### GitLab self-monitoring project @@ -141,7 +260,10 @@ changes to your code, settings, or workflow. GitLab self-monitoring gives administrators of self-hosted GitLab instances the tools to monitor the health of their instances. This feature is deprecated in GitLab 14.9, and is scheduled for removal in 16.0. -**Planned removal milestone: 16.0 (2023-05-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### GraphQL permissions change for Package settings @@ -160,18 +282,24 @@ The permissions model for GraphQL is being updated. After 15.0, users with the G - [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) - [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting) -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and will be removed from GitLab Shell in GitLab 15.0. -**Planned removal milestone: 15.0 ()** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-1410 breaking-change"> ### Permissions change for downloading Composer dependencies WARNING: -This feature will be changed or removed in 15.0 +This feature will be changed or removed in 14.10 as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. @@ -180,7 +308,10 @@ The GitLab Composer repository can be used to push, search, fetch metadata about Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">14.10</span> (2022-04-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### htpasswd Authentication for the Container Registry @@ -194,7 +325,10 @@ The Container Registry supports [authentication](https://gitlab.com/gitlab-org/c Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### user_email_lookup_limit API field @@ -208,21 +342,15 @@ The `user_email_lookup_limit` [API field](https://docs.gitlab.com/ee/api/setting Any API calls attempting to change the rate limits for `user_email_lookup_limit` should use `search_rate_limit` instead. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> -## 14.8 - -### Changes to the `CI_JOB_JWT` - -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +<div class="announcement-milestone"> -The `CI_JOB_JWT` will be updated to support a wider variety of cloud providers. It will be changed to match [`CI_JOB_JWT_V2`](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html), but this change may not be backwards compatible for all users, including Hashicorp Vault users. To maintain the current behavior, users can switch to using `CI_JOB_JWT_V1`, or update their configuration in GitLab 15.0 to use the improved `CI_JOB_JWT`. +## 14.8 -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-149"> ### Configurable Gitaly `per_repository` election strategy @@ -231,7 +359,10 @@ Configuring the `per_repository` Gitaly election strategy is [deprecated](https: This change is part of regular maintenance to keep our codebase clean. -**Planned removal milestone: 14.9 (2022-03-22)** +**Planned removal milestone: <span class="removal-milestone">14.9</span> (2022-03-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Container Network and Host Security @@ -241,7 +372,7 @@ as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#brea Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through the GitLab [Secure CI/CD Tunnel](https://docs.gitlab.com/ee/user/clusters/agent/repository.html#run-kubectl-commands-using-the-cicd-tunnel). +All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). As part of this change, the following specific capabilities within GitLab are now deprecated, and are scheduled for removal in GitLab 15.0: @@ -252,7 +383,10 @@ As part of this change, the following specific capabilities within GitLab are no For additional context, or to provide feedback regarding this change, please reference our open [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7476). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Dependency Scanning Python 3.9 and 3.6 image deprecation @@ -282,13 +416,19 @@ gemnasium-python-dependency_scanning: name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 ``` -**Planned removal milestone: 15.0 (2021-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** +</div> + +<div class="deprecation removal-150"> ### Deprecate Geo Admin UI Routes In GitLab 13.0, we introduced new project and design replication details routes in the Geo Admin UI. These routes are `/admin/geo/replication/projects` and `/admin/geo/replication/designs`. We kept the legacy routes and redirected them to the new routes. In GitLab 15.0, we will remove support for the legacy routes `/admin/geo/projects` and `/admin/geo/designs`. Please update any bookmarks or scripts that may use the legacy routes. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Deprecate custom Geo:db:* Rake tasks @@ -313,7 +453,10 @@ The following `geo:db:*` tasks will be replaced with their corresponding `db:*:g - `geo:db:test:load` -> `db:test:load:geo` - `geo:db:test:purge` -> `db:test:purge:geo` -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Deprecate feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS @@ -325,7 +468,10 @@ changes to your code, settings, or workflow. The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede CODEOWNERS. The CODEOWNERS feature will no longer be available for access control. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Deprecate legacy Gitaly configuration methods @@ -341,7 +487,10 @@ These variables are being replaced with standard [`config.toml` Gitaly configura GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using `config.toml`. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Elasticsearch 6.8 @@ -357,7 +506,10 @@ We recommend using the latest version of Elasticsearch 7 to benefit from all Ela Elasticsearch 6.8 is also incompatible with Amazon OpenSearch, which we [plan to support in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/327560). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### External status check API breaking changes @@ -383,7 +535,10 @@ and set to `passed`. Requests that: To align with this change, API calls to list external status checks will also return the value of `passed` rather than `approved` for status checks that have passed. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### GraphQL ID and GlobalID compatibility @@ -445,7 +600,10 @@ You should convert any queries in the first form (using `ID` as a named type in to one of the other two forms (using the correct appropriate type in the signature, or using an inline argument expression). -**Planned removal milestone: 15.0 (2022-04-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-04-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### OAuth tokens without expiration @@ -465,7 +623,10 @@ tokens before GitLab 15.0 is released: 1. Edit the application. 1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Optional enforcement of PAT expiration @@ -479,7 +640,10 @@ The feature to disable enforcement of PAT expiration is unusual from a security We have become concerned that this unusual feature could create unexpected behavior for users. Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Optional enforcement of SSH expiration @@ -493,7 +657,10 @@ The feature to disable enforcement of SSH expiration is unusual from a security We have become concerned that this unusual feature could create unexpected behavior for users. Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Out-of-the-box SAST support for Java 8 @@ -516,7 +683,10 @@ In GitLab 15.0, we will: If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Querying Usage Trends via the `instanceStatisticsMeasurements` GraphQL node @@ -528,7 +698,10 @@ changes to your code, settings, or workflow. The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTrendsMeasurements` in 13.10 and the old field name has been marked as deprecated. To fix the existing GraphQL queries, replace `instanceStatisticsMeasurements` with `usageTrendsMeasurements`. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### REST API Runner will not accept `status` filter values of `active` or `paused` @@ -546,7 +719,10 @@ Status values `paused` or `active` will no longer be accepted and will be replac When checking for paused runners, API users are advised to specify `paused=true` as the query parameter. When checking for active runners, specify `paused=false`. -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### REST API endpoint to list group runners no longer accepts `project_type` value for `type` argument @@ -558,7 +734,10 @@ changes to your code, settings, or workflow. The `GET /groups/:id/runners?type=project_type` endpoint will be removed in GitLab 16.0. The endpoint always returned an empty collection. -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### REST and GraphQL API Runner usage of `active` replaced by `paused` @@ -588,7 +767,10 @@ The 16.0 release of the GitLab Runner will start using the `paused` property whe will only be compatible with GitLab 16.0 and later. Until 16.0, GitLab will accept the deprecated `active` flag from existing runners. -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Request profiling @@ -606,7 +788,10 @@ It also depends on a few third-party gems that are not actively maintained anymo For more information, check the [summary section of the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488#deprecation-summary). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Required pipeline configurations in Premium tier @@ -623,7 +808,10 @@ This change to move the feature to GitLab's Ultimate tier is intended to help ou This change will also help GitLab remain consistent in its tiering strategy with the other related Ultimate-tier features of: [Security policies](https://docs.gitlab.com/ee/user/application_security/policies/) and [compliance framework pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Retire-JS Dependency Scanning tool @@ -637,12 +825,15 @@ As of 14.8 the retire.js job is being deprecated from Dependency Scanning. It wi If you have explicitly excluded retire.js using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration related to the `retire-js-dependency_scanning` job you will want to switch to gemnasium-dependency_scanning before the removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference retire.js, or customized your template specifically for retire.js, you will not need to take action. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-152 breaking-change"> ### SAST analyzer consolidation and CI/CD template changes WARNING: -This feature will be changed or removed in 15.0 +This feature will be changed or removed in 15.2 as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. @@ -652,12 +843,15 @@ GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application We are reducing the number of analyzers used in GitLab SAST as part of our long-term strategy to deliver a better and more consistent user experience. Streamlining the set of analyzers will also enable faster [iteration](https://about.gitlab.com/handbook/values/#iteration), better [results](https://about.gitlab.com/handbook/values/#results), and greater [efficiency](https://about.gitlab.com/handbook/values/#results) (including a reduction in CI runner usage in most cases). -In GitLab 15.0, GitLab SAST will no longer use the following analyzers: +In GitLab 15.2, GitLab SAST will no longer use the following analyzers: - [ESLint](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) (JavaScript, TypeScript, React) - [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) (Go) - [Bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) (Python) +NOTE: +This change was originally planned for GitLab 15.0 and has been postponed. + These analyzers will be removed from the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) and replaced with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). They will no longer receive routine updates, except for security issues. We will not delete container images previously published for these analyzers; any such change would be announced as a [deprecation, removal, or breaking change announcement](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations-removals-and-breaking-changes). @@ -666,9 +860,12 @@ We will also remove Java from the scope of the [SpotBugs](https://gitlab.com/git This change will make it simpler to scan Java code; compilation will no longer be required. This change will be reflected in the automatic language detection portion of the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml). -If you applied customizations to any of the affected analyzers, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352554#breaking-change). +If you applied customizations to any of the affected analyzers or if you currently disable the Semgrep analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352554#breaking-change). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.2</span> (2022-07-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### SAST support for .NET 2.1 @@ -698,7 +895,10 @@ Version 3 was [announced in GitLab 14.6](https://about.gitlab.com/releases/2021/ If you rely on .NET 2.1 support being present in the analyzer image by default, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352553#breaking-change). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Secret Detection configuration variables deprecated @@ -719,7 +919,10 @@ You'll still be able to configure historical scanning of your commit history by For further details, see [the deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352565). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Secure and Protect analyzer images published in new location @@ -745,7 +948,10 @@ Otherwise, you won't receive further updates. See the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564) for more details. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Secure and Protect analyzer major version update @@ -788,7 +994,10 @@ Specifically, the following are being deprecated and will no longer be updated a - `sobelow`: version 2 - `spotbugs`: version 2 -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Support for gRPC-aware proxy deployed between Gitaly and rest of GitLab @@ -810,7 +1019,10 @@ By sending some of our internal RPC traffic through a custom protocol (instead o increase throughput and reduce Go garbage collection latency. For more information, see the [relevant epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/463). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Test coverage project CI/CD setting @@ -821,13 +1033,16 @@ Before updating GitLab, review the details carefully to determine if you need to changes to your code, settings, or workflow. To simplify setting a test coverage pattern, in GitLab 15.0 the -[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-using-project-settings-deprecated) +[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-using-project-settings-removed) is being removed. Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set testing coverage results in merge requests. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Vulnerability Check @@ -846,7 +1061,10 @@ The new security approvals feature is similar to vulnerability check. For exampl - A two-step approval process can be enforced for any desired changes to security approval rules. - A single set of security policies can be applied to multiple development projects to allow for ease in maintaining a single, centralized ruleset. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### `CI_BUILD_*` predefined variables @@ -873,7 +1091,10 @@ The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in G | `CI_BUILD_TOKEN` | `CI_JOB_TOKEN` | | `CI_BUILD_TRIGGERED` | `CI_PIPELINE_TRIGGERED` | -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-150"> ### `fixup!` commit messages setting draft status of associated Merge Request @@ -885,7 +1106,10 @@ messages, as part of our streamlining of the feature. Support for `fixup!` is now considered deprecated, and will be removed in GitLab 15.0. -**Planned removal milestone: 15.0 (2022-06-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-06-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `projectFingerprint` in `PipelineSecurityReportFinding` GraphQL @@ -900,7 +1124,10 @@ GraphQL object is being deprecated. This field contains a "fingerprint" of secur The method for calculating fingerprints has changed, resulting in different values. Going forward, the new values will be exposed in the UUID field. Data previously available in the projectFingerprint field will eventually be removed entirely. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `started` iterations API field @@ -912,10 +1139,16 @@ changes to your code, settings, or workflow. The `started` field in the [iterations API](https://docs.gitlab.com/ee/api/iterations.html#list-project-iterations) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `current` field (already available) which aligns with the naming for other time-based entities, such as milestones. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.7 +<div class="deprecation removal-150"> + ### Container scanning schemas below 14.0.0 [Container scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) @@ -929,7 +1162,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Coverage guided fuzzing schemas below 14.0.0 @@ -947,7 +1183,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### DAST schemas below 14.0.0 @@ -965,7 +1204,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will cause [warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Dependency scanning schemas below 14.0.0 @@ -983,7 +1225,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will cause [warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Enforced validation of security report schemas @@ -1001,7 +1246,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Godep support in License Compliance @@ -1010,7 +1258,10 @@ has been replaced with Go modules. To reduce our maintenance cost we are deprecating License Compliance for Godep projects as of 14.7 and will remove it in GitLab 15.0 -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Logging in GitLab @@ -1022,7 +1273,10 @@ changes to your code, settings, or workflow. The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users can search for relevant logs in GitLab. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, we don't have a recommended solution for logging within GitLab. For more information, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### Monitor performance metrics through Prometheus @@ -1035,7 +1289,10 @@ changes to your code, settings, or workflow. By displaying data stored in a Prometheus instance, GitLab allows users to view performance metrics. GitLab also displays visualizations of these metrics in dashboards. The user can connect to a previously-configured external Prometheus instance, or set up Prometheus as a GitLab Managed App. However, since certificate-based integration with Kubernetes clusters is deprecated in GitLab, the metrics functionality in GitLab that relies on Prometheus is also deprecated. This includes the metrics visualizations in dashboards. GitLab is working to develop a single user experience based on [Opstrace](https://about.gitlab.com/press/releases/2021-12-14-gitlab-acquires-opstrace-to-expand-its-devops-platform-with-open-source-observability-solution.html). An [issue exists](https://gitlab.com/groups/gitlab-org/-/epics/6976) for you to follow work on the Opstrace integration. -**Planned removal milestone: 16.0 (2023-05-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-150"> ### Pseudonymizer @@ -1044,7 +1301,10 @@ can cause production issues with large databases, and can interfere with object storage development. It is now considered deprecated, and will be removed in GitLab 15.0. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### SAST schemas below 14.0.0 @@ -1062,7 +1322,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Secret detection schemas below 14.0.0 @@ -1080,7 +1343,10 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Sidekiq metrics and health checks configuration @@ -1110,13 +1376,21 @@ and only run one server (not changing the current behaviour). Only if they are both set and a different port is provided, a separate metrics server will spin up to serve the Sidekiq metrics, similar to the way Sidekiq will behave in 15.0. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### Static Site Editor -The Static Site Editor will no longer be available starting in GitLab 15.0. Improvements to the Markdown editing experience across GitLab will deliver smiliar benefit but with a wider reach. Incoming requests to the Static Site Editor will be redirected to the Web IDE. Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/static_site_editor/) for more information, including how to remove the configuration files from existing projects. +The Static Site Editor will no longer be available starting in GitLab 15.0. Improvements to the Markdown editing experience across GitLab will deliver smiliar benefit but with a wider reach. Incoming requests to the Static Site Editor will be redirected to the [Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/index.html). + +Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/static_site_editor/) for more information, including how to remove the configuration files from existing projects. + +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-150 breaking-change"> ### Tracing in GitLab @@ -1128,7 +1402,10 @@ changes to your code, settings, or workflow. Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users can navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab is deprecated in GitLab 14.7, and scheduled for removal in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150"> ### `artifacts:report:cobertura` keyword @@ -1137,7 +1414,10 @@ Currently, test coverage visualizations in GitLab only support Cobertura reports [`artifacts:reports:coverage_report`](https://gitlab.com/gitlab-org/gitlab/-/issues/344533). Cobertura will be the only supported report file in 15.0, but this is the first step towards GitLab supporting other report types. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### merged_by API field @@ -1149,23 +1429,15 @@ changes to your code, settings, or workflow. The `merged_by` field in the [merge request API](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `merge_user` field (already present in GraphQL) which more correctly identifies who merged a merge request when performing actions (merge when pipeline succeeds, add to merge train) other than a simple merge. -**Planned removal milestone: 15.0 (2022-05-22)** - -## 14.6 +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> -### API: `stale` status returned instead of `offline` or `not_connected` +<div class="announcement-milestone"> -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -A breaking change will occur for the Runner [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints in 15.0. - -Instead of the GitLab Runner API endpoints returning `offline` and `not_connected` for runners that have not contacted the GitLab instance in the past three months, the API endpoints will return the `stale` value, which was introduced in 14.6. +## 14.6 -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-150 breaking-change"> ### CI/CD job name length limit @@ -1177,7 +1449,10 @@ changes to your code, settings, or workflow. In GitLab 15.0 we are going to limit the number of characters in CI/CD job names to 255. Any pipeline with job names that exceed the 255 character limit will stop working after the 15.0 release. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Legacy approval status names from License Compliance API @@ -1191,36 +1466,10 @@ We deprecated legacy names for approval status of license policy (blacklisted, a If you are using our License Compliance API you should stop using the `approved` and `blacklisted` query parameters, they are now `allowed` and `denied`. In 15.0 the responses will also stop using `approved` and `blacklisted` so you need to adjust any of your custom tools to use the old and new values so they do not break with the 15.0 release. -**Planned removal milestone: 15.0 (2022-05-22)** - -### Runner status `not_connected` API value - -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -The GitLab Runner REST and GraphQL [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints -will return `never_contacted` instead of `not_connected` as the status values in 15.0. +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> -Runners that have never contacted the GitLab instance will also return `stale` if created more than 3 months ago. - -**Planned removal milestone: 15.0 (2022-05-22)** - -### `pipelines` fields in the Package GraphQL types - -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `pipelines` fields in all Package-related GraphQL types. As of GitLab 14.6, the `pipelines` field is deprecated in [`Package`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#package) and [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype) due to scalability and performance concerns. - -In milestone 15.0, we will completely remove `pipelines` from `Package` and `PackageDetailsType`. You can follow and contribute to work on a replacement in the epic [GitLab-#7214](https://gitlab.com/groups/gitlab-org/-/epics/7214). - -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-150 breaking-change"> ### `type` and `types` keyword in CI/CD configuration @@ -1232,7 +1481,10 @@ changes to your code, settings, or workflow. The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines that use these keywords will stop working, so you must switch to `stage` and `stages`, which have the same behavior. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### apiFuzzingCiConfigurationCreate GraphQL mutation @@ -1246,7 +1498,10 @@ The API Fuzzing configuration snippet is now being generated client-side and doe API request anymore. We are therefore deprecating the `apiFuzzingCiConfigurationCreate` mutation which isn't being used in GitLab anymore. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### bundler-audit Dependency Scanning tool @@ -1260,48 +1515,34 @@ As of 14.6 bundler-audit is being deprecated from Dependency Scanning. It will c If you have explicitly excluded bundler-audit using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration, for example to edit the `bundler-audit-dependency_scanning` job, you will want to switch to gemnasium-dependency_scanning before removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference bundler-audit, or customized your template specifically for bundler-audit, you will not need to take action. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.5 -### Certificate-based integration with Kubernetes +<div class="deprecation removal-150 breaking-change"> + +### Changing an instance (shared) runner to a project (specific) runner WARNING: -This feature will be changed or removed in 15.6 +This feature will be changed or removed in 15.0 as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -[The certificate-based integration with Kubernetes will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). - -If you are a self-managed customer, in GitLab 15.0, a feature flag will be introduced so you can keep -certificate-based integration enabled. The flag will be disabled by default. -The flag and the related code will be removed in GitLab 15.6. - -Until the final removal in 15.6, features built on the integration will continue to work, and -GitLab will continue to fix security and critical issues. - -If you use GitLab.com, certificate-based integrations will cease functioning in 15.0. - -For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the -[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. -See the documentation for [how to migrate](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html). - -For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). - -**Planned removal milestone: 15.6 (2022-11-22)** +In GitLab 15.0, you can no longer change an instance (shared) runner to a project (specific) runner. -### Converting an instance (shared) runner to a project (specific) runner +Users often accidentally change instance runners to project runners, and they're unable to change them back. GitLab does not allow you to change a project runner to a shared runner because of the security implications. A runner meant for one project could be set to run jobs for an entire instance. -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +Administrators who need to add runners for multiple projects can register a runner for one project, then go to the Admin view and choose additional projects. -In GitLab 15.0, we will remove the feature that enables you to convert an instance (shared) runner to a project (specific) runner. Users who need to add a runner to only a particular project can register a runner to the project directly. +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-150 breaking-change"> ### Known host required for GitLab Runner SSH executor @@ -1315,7 +1556,10 @@ In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/30 In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Must explicitly assign `AuthenticationType` for `[runners.cache.s3]` @@ -1329,21 +1573,27 @@ In GitLab 15.0 and later, to access the AWS S3 cache, you must specify the `Auth Prior to 14.5, if you did not define the `AuthenticationType`, GitLab Runner chose a type for you. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> ### Package pipelines in API payload is paginated WARNING: -This feature will be changed or removed in 15.0 +This feature will be changed or removed in 16.0 as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. A request to the API for `/api/v4/projects/:id/packages` returns a paginated result of packages. Each package lists all of its pipelines in this response. This is a performance concern, as it's possible for a package to have hundreds or thousands of associated pipelines. -In milestone 15.0, we will remove the `pipelines` attribute from the API response. +In milestone 16.0, we will remove the `pipelines` attribute from the API response. + +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> -**Planned removal milestone: 15.0 (2022-05-22)** +<div class="deprecation removal-160 breaking-change"> ### REST and GraphQL API Runner status will not return `paused` @@ -1361,7 +1611,56 @@ A runner's status will only relate to runner contact status, such as: When checking if a runner is `paused`, API users are advised to check the boolean attribute `paused` to be `true` instead. When checking if a runner is `active`, check if `paused` is `false`. -**Planned removal milestone: 16.0 (2023-04-22)** +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** +</div> + +<div class="deprecation removal-156 breaking-change"> + +### SaaS certificate-based integration with Kubernetes + +WARNING: +This feature will be changed or removed in 15.6 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The certificate-based integration with Kubernetes will be [deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). As a GitLab SaaS customer, on new namespaces, you will no longer be able to integrate GitLab and your cluster using the certificate-based approach as of GitLab 15.0. The integration for current users will be enabled per namespace. The integrations are expected to be switched off completely on GitLab SaaS around 2022 November 22. + +For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the +[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) + +For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). + +GitLab self-managed customers can still use the feature [with a feature flag](https://docs.gitlab.com/ee/update/deprecations.html#self-managed-certificate-based-integration-with-kubernetes). + +**Planned removal milestone: <span class="removal-milestone">15.6</span> (2022-11-22)** +</div> + +<div class="deprecation removal-160 breaking-change"> + +### Self-managed certificate-based integration with Kubernetes + +WARNING: +This feature will be changed or removed in 16.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The certificate-based integration with Kubernetes [will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). + +As a self-managed customer, we are introducing a feature flag in GitLab 15.0 so you can keep your certificate-based integration enabled. However, the feature flag will be disabled by default, so this change is a **breaking change**. + +In GitLab 16.0 we will remove both the feature and its related code. Until the final removal in 16.0, features built on this integration will continue to work, if you enable the feature flag. Until the feature is removed, GitLab will continue to fix security and critical issues as they arise. + +For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the +[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) + +For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). + +**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Support for SLES 12 SP2 @@ -1373,7 +1672,10 @@ changes to your code, settings, or workflow. Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Update to the Container Registry group-level API @@ -1387,7 +1689,10 @@ In milestone 15.0, support for the `tags` and `tags_count` parameters will be re The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Value Stream Analytics filtering calculation change @@ -1401,7 +1706,10 @@ We are changing how the date filter works in Value Stream Analytics. Instead of If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `Versions` on base `PackageType` @@ -1415,7 +1723,10 @@ As part of the work to create a [Package Registry GraphQL API](https://gitlab.co In milestone 15.0, we will completely remove `Version` from `PackageType`. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `defaultMergeCommitMessageWithDescription` GraphQL API field @@ -1427,7 +1738,10 @@ changes to your code, settings, or workflow. The GraphQL API field `defaultMergeCommitMessageWithDescription` has been deprecated and will be removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `dependency_proxy_for_private_groups` feature flag @@ -1441,7 +1755,10 @@ We added a feature flag because [GitLab-#11582](https://gitlab.com/gitlab-org/gi In milestone 15.0, we will remove the feature flag entirely. Moving forward, you must authenticate when using the Dependency Proxy. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `pipelines` field from the `version` field @@ -1458,7 +1775,10 @@ In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDeta To mitigate possible performance problems, we will remove the `versions` field's `pipelines` field in milestone 15.0. Although you will no longer be able to get all pipelines for all versions of a package, you can still get the pipelines of a single version through the remaining `pipelines` field for that version. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `promote-db` command from `gitlab-ctl` @@ -1470,7 +1790,10 @@ changes to your code, settings, or workflow. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. `gitlab-ctl promote-db` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### `promote-to-primary-node` command from `gitlab-ctl` @@ -1482,7 +1805,10 @@ changes to your code, settings, or workflow. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-148"> ### openSUSE Leap 15.2 packages @@ -1490,10 +1816,16 @@ Distribution support and security updates for openSUSE Leap 15.2 are [ending Dec Starting in 14.5 we are providing packages for openSUSE Leap 15.3, and will stop providing packages for openSUSE Leap 15.2 in the 14.8 milestone. -**Planned removal milestone: 14.8 (2022-02-22)** +**Planned removal milestone: <span class="removal-milestone">14.8</span> (2022-02-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.3 +<div class="deprecation removal-150 breaking-change"> + ### Audit events for repository push events WARNING: @@ -1502,13 +1834,16 @@ as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#brea Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#repository-push-deprecated) are now deprecated and will be removed in GitLab 15.0. +Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are now deprecated and will be removed in GitLab 15.0. These events have always been disabled by default and had to be manually enabled with a feature flag. Enabling them can cause too many events to be generated which can dramatically slow down GitLab instances. For this reason, they are being removed. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### GitLab Serverless @@ -1522,7 +1857,10 @@ changes to your code, settings, or workflow. We decided to remove the GitLab Serverless features as they never really resonated with our users. Besides, given the continuous development of Kubernetes and Knative, our current implementations do not even work with recent versions. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### Legacy database configuration @@ -1538,7 +1876,10 @@ supported using a single PostgreSQL adapter, whereas the new format is changing This deprecation mainly impacts users compiling GitLab from source because Omnibus will handle this configuration automatically. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### OmniAuth Kerberos gem @@ -1554,15 +1895,24 @@ This gem has not been maintained and has very little usage. We therefore plan to Note that we are not deprecating the Kerberos SPNEGO integration, only the old password-based Kerberos integration. -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.2 +<div class="deprecation removal-146"> + ### Release CLI distributed as a generic package The [release-cli](https://gitlab.com/gitlab-org/release-cli) will be released as a [generic package](https://gitlab.com/gitlab-org/release-cli/-/packages) starting in GitLab 14.2. We will continue to deploy it as a binary to S3 until GitLab 14.5 and stop distributing it in S3 in GitLab 14.6. -**Planned removal milestone: 14.6 (2021-12-22)** +**Planned removal milestone: <span class="removal-milestone">14.6</span> (2021-12-22)** +</div> + +<div class="deprecation removal-145"> ### Rename Task Runner pod to Toolbox @@ -1570,10 +1920,16 @@ The Task Runner pod is used to execute periodic housekeeping tasks within the Gi This will result in the rename of the sub-chart: `gitlab/task-runner` to `gitlab/toolbox`. Resulting pods will be named along the lines of `{{ .Release.Name }}-toolbox`, which will often be `gitlab-toolbox`. They will be locatable with the label `app=toolbox`. -**Planned removal milestone: 14.5 (2021-11-22)** +**Planned removal milestone: <span class="removal-milestone">14.5</span> (2021-11-22)** +</div> +</div> + +<div class="announcement-milestone"> ## 14.0 +<div class="deprecation removal-156"> + ### NFS for Git repository storage With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for further information. @@ -1586,7 +1942,10 @@ Gitaly Cluster offers tremendous benefits for our customers such as: We encourage customers currently using NFS for Git repositories to plan their migration by reviewing our documentation on [migrating to Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/index.html#migrate-to-gitaly-cluster). -**Planned removal milestone: 15.6 (2022-11-22)** +**Planned removal milestone: <span class="removal-milestone">15.6</span> (2022-11-22)** +</div> + +<div class="deprecation removal-150 breaking-change"> ### OAuth implicit grant @@ -1598,4 +1957,6 @@ changes to your code, settings, or workflow. The OAuth implicit grant authorization flow will be removed in our next major release, GitLab 15.0. Any applications that use OAuth implicit grant should switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). -**Planned removal milestone: 15.0 (2022-05-22)** +**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** +</div> +</div> diff --git a/doc/update/index.md b/doc/update/index.md index a21b55b0205..24afb01396a 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -193,8 +193,12 @@ pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_cla GitLab 13.6 introduced an issue where a background migration named `BackfillJiraTrackerDeploymentType2` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [13.6.0 version-specific instructions](#1360). +GitLab 14.2 introduced an issue where a background migration named `BackfillDraftStatusOnMergeRequests` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.2.0 version-specific instructions](#1420). + GitLab 14.4 introduced an issue where a background migration named `PopulateTopicsTotalProjectsCountCache` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.4.0 version-specific instructions](#1440). +GitLab 14.5 introduced an issue where a background migration named `UpdateVulnerabilityOccurrencesLocation` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.5.0 version-specific instructions](#1450). + GitLab 14.8 introduced an issue where a background migration named `PopulateTopicsNonPrivateProjectsCount` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [14.8.0 version-specific instructions](#1480). GitLab 14.9 introduced an issue where a background migration named `ResetDuplicateCiRunnersTokenValuesOnProjects` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.9.0 version-specific instructions](#1490). @@ -325,7 +329,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab accordingly, while also consulting the [version-specific upgrade instructions](#version-specific-upgrading-instructions): -`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> `12.10.14` -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [latest `14.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) +`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.9.0`](#1490) -> [latest `14.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) The following table, while not exhaustive, shows some examples of the supported upgrade paths. @@ -402,6 +406,17 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 15.0.0 + +- Elasticsearch 6.8 [is no longer supported](../integration/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version). + +### 14.10.0 + +- Before upgrading to GitLab 14.10, you need to already have the latest 14.9.Z installed on your instance. + The upgrade to GitLab 14.10 executes a [concurrent index drop](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84308) of unneeded + entries from the `ci_job_artifacts` database table. This could potentially run for multiple minutes, especially if the table has a lot of + traffic and the migration is unable to acquire a lock. It is advised to let this process finish as restarting may result in data loss. + ### 14.9.0 - Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. @@ -418,6 +433,17 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ```plaintext Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': ``` + + Or + + ```plaintext + Error executing action `run` on resource 'bash[migrate gitlab-rails database]' + ================================================================================ + + Mixlib::ShellOut::ShellCommandFailed + ------------------------------------ + Command execution failed. STDOUT/STDERR suppressed for sensitive resource + ``` - GitLab 14.9.0 includes a [background migration `ResetDuplicateCiRunnersTokenValuesOnProjects`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79140) @@ -462,7 +488,7 @@ that may remain stuck permanently in a **pending** state. ```ruby Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "PopulateTopicsNonPrivateProjectsCount").find_each do |job| - puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsNonPrivateProjectsCountq", job.arguments) + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsNonPrivateProjectsCount", job.arguments) end ``` @@ -525,10 +551,22 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo For more information, refer to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331823). +- GitLab 14.5.0 includes a + [background migration `UpdateVulnerabilityOccurrencesLocation`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72788) + that may remain stuck permanently in a **pending** state when the instance lacks records that match the migration's target. + + To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): + + ```ruby + Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "UpdateVulnerabilityOccurrencesLocation").find_each do |job| + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("UpdateVulnerabilityOccurrencesLocation", job.arguments) + end + ``` + ### 14.4.4 - For [zero-downtime upgrades](zero_downtime.md) on a GitLab cluster with separate Web and API nodes, you need to enable the `paginated_tree_graphql_query` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) _before_ upgrading GitLab Web nodes to 14.4. - This is because we [enabled `paginated_tree_graphql_query by default in 14.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70913/diffs), so if GitLab UI is on 14.4 and its API is on 14.3, the frontend will have this feature enabled but the backend will have it disabled. This will result in the following error: + This is because we [enabled `paginated_tree_graphql_query` by default in 14.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70913/diffs), so if GitLab UI is on 14.4 and its API is on 14.3, the frontend will have this feature enabled but the backend will have it disabled. This will result in the following error: ```shell bundle.esm.js:63 Uncaught (in promise) Error: GraphQL error: Field 'paginatedTree' doesn't exist on type 'Repository' @@ -610,6 +648,17 @@ for how to proceed. ``` - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). +- GitLab 14.2.0 includes a + [background migration `BackfillDraftStatusOnMergeRequests`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67687) + that may remain stuck permanently in a **pending** state when the instance lacks records that match the migration's target. + + To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): + + ```ruby + Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "BackfillDraftStatusOnMergeRequests").find_each do |job| + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("BackfillDraftStatusOnMergeRequests", job.arguments) + end + ``` ### 14.1.0 @@ -630,6 +679,16 @@ for how to proceed. ### 14.0.0 +Prerequisites: + +- The [GitLab 14.0 release post contains several important notes](https://about.gitlab.com/releases/2021/06/22/gitlab-14-0-released/#upgrade) + about pre-requisites including [using Patroni instead of repmgr](../administration/postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni), + migrating [to hashed storage](../administration/raketasks/storage.md#migrate-to-hashed-storage), + and [to Puma](../administration/operations/puma.md). +- The support of PostgreSQL 11 [has been dropped](../install/requirements.md#database). Make sure to [update your database](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server) to version 12 before updating to GitLab 14.0. + +Long running batched background database migrations: + - Database changes made by the upgrade to GitLab 14.0 can take hours or days to complete on larger GitLab instances. These [batched background migrations](#batched-background-migrations) update whole database tables to mitigate primary key overflow and must be finished before upgrading to GitLab 14.2 or higher. - Due to an issue where `BatchedBackgroundMigrationWorkers` were @@ -650,13 +709,12 @@ for how to proceed. See how to [resolve this error](../user/admin_area/monitoring/background_migrations.md#database-migrations-failing-because-of-batched-background-migration-not-finished). +Other issues: + - In GitLab 13.3 some [pipeline processing methods were deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/218536) and this code was completely removed in GitLab 14.0. If you plan to upgrade from - **GitLab 13.2 or older** directly to 14.0 ([unsupported](#upgrading-to-a-new-major-version)), you should not have any pipelines running - when you upgrade or the pipelines might report the wrong status when the upgrade completes. + **GitLab 13.2 or older** directly to 14.0, this is [unsupported](#upgrading-to-a-new-major-version). You should instead follow a [supported upgrade path](#upgrade-paths). -- The support of PostgreSQL 11 [has been dropped](../install/requirements.md#database). Make sure to [update your database](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server) to version 12 before updating to GitLab 14.0. - - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). - See [Custom Rack Attack initializers](#custom-rack-attack-initializers) if you persist your own custom Rack Attack initializers during upgrades. @@ -673,7 +731,16 @@ for how to proceed. ### 13.12.0 -See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). +- See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). + +- Check the GitLab database [has no references to legacy storage](../administration/raketasks/storage.md#on-legacy-storage). + The GitLab 14.0 pre-install check will cause the package update to fail if there is unmigrated data: + + ```plaintext + Checking for unmigrated data on legacy storage + + Legacy storage is no longer supported. Please migrate your data to hashed storage. + ``` ### 13.11.0 @@ -846,6 +913,20 @@ supplied with GitLab during upgrades. We recommend you use these GitLab-supplied If you persist your own Rack Attack initializers between upgrades, you might [get `500` errors](https://gitlab.com/gitlab-org/gitlab/-/issues/334681) when [upgrading to GitLab 14.0 and later](#1400). +### 12.10.0 + +- The final patch release (12.10.14) + [has a regression affecting maven package uploads](https://about.gitlab.com/releases/2020/07/06/critical-security-release-gitlab-13-1-3-released/#maven-package-upload-broken-in-121014). + If you use this feature and need to stay on 12.10 while preparing to upgrade to 13.0: + + - Upgrade to 12.10.13 instead. + - Upgrade to 13.0.14 as soon as possible. + +- [GitLab 13.0 requires PostgreSQL 11](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab). + + - 12.10 is the final release that shipped with PostgreSQL 9.6, 10, and 11. + - You should make sure that your database is PostgreSQL 11 on GitLab 12.10 before upgrading to 13.0. This will require downtime. + ### 12.2.0 In 12.2.0, we enabled Rails' authenticated cookie encryption. Old sessions are @@ -884,7 +965,7 @@ for more information. When [Maintenance mode](../administration/maintenance_mode/index.md) is enabled, users cannot sign in with SSO, SAML, or LDAP. -Users who were signed in before Maintenance mode was enabled will continue to be signed in. If the admin who enabled Maintenance mode loses their session, then they will not be able to disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/#disable-maintenance-mode). +Users who were signed in before Maintenance mode was enabled will continue to be signed in. If the administrator who enabled Maintenance mode loses their session, then they will not be able to disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/#disable-maintenance-mode). [This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/329261) was fixed in GitLab 14.5.0 and backported into 14.4.3 and 14.3.5. diff --git a/doc/update/package/convert_to_ee.md b/doc/update/package/convert_to_ee.md index d5a71ba3e80..8a0d55e34af 100644 --- a/doc/update/package/convert_to_ee.md +++ b/doc/update/package/convert_to_ee.md @@ -91,7 +91,7 @@ The steps can be summed up to: sudo gitlab-ctl reconfigure ``` -1. Now go to the GitLab admin panel of your server (`/admin/subscription`) and +1. Now go to the GitLab Admin Area of your server (`/admin/subscription`) and [add your license](../../user/admin_area/license.md). 1. After you confirm that GitLab is working as expected, you may remove the old diff --git a/doc/update/package/index.md b/doc/update/package/index.md index b6417e10917..faca633f446 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -37,6 +37,7 @@ GitLab package. Upgrading versions might need some manual intervention. For more information, check the version your are upgrading to: +- [GitLab 15](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html) - [GitLab 14](https://docs.gitlab.com/omnibus/update/gitlab_14_changes.html) - [GitLab 13](https://docs.gitlab.com/omnibus/update/gitlab_13_changes.html) - [GitLab 12](https://docs.gitlab.com/omnibus/update/gitlab_12_changes.html) @@ -126,7 +127,9 @@ or upgrade command: ``` 1. Install the specific `gitlab-ee` package by using one of the following commands - and replacing `<version>` with the version you found in the previous step: + and replacing `<version>` with the next supported version you would like to install + (make sure to review the [upgrade path](../index.md#upgrade-paths) to confirm the + version you're installing is part of a supported path): ```shell # Ubuntu/Debian @@ -296,3 +299,12 @@ To fix this issue: ### Error `Failed to connect to the internal GitLab API` on a separate GitLab Pages server Please see [GitLab Pages troubleshooting](../../administration/pages/index.md#failed-to-connect-to-the-internal-gitlab-api). + +### Error `An error occurred during the signature verification` when running `apt-get update` + +To update the GPG key of the GitLab packages server run: + +```shell +curl --silent "https://packages.gitlab.com/gpg.key" | apt-key add - +apt-get update +``` diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 6eebfd1b278..99812e7fdb2 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -14,7 +14,7 @@ General notes: - If possible, we recommend you test out the upgrade in a test environment before updating your production instance. Ideally, your test environment should mimic your production environment as closely as possible. -- If [working with Support](https://about.gitlab.com/support/scheduling-live-upgrade-assistance.html) +- If [working with Support](https://about.gitlab.com/support/scheduling-upgrade-assistance.html) to create your plan, share details of your architecture, including: - How is GitLab installed? - What is the operating system of the node? diff --git a/doc/update/removals.md b/doc/update/removals.md index 7e2b4f84fa1..4e653d5ab0a 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -28,6 +28,694 @@ For removal reviewers (Technical Writers only): https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-removals-doc --> +## 15.0 + +### API: `stale` status returned instead of `offline` or `not_connected` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +A breaking change was made to the Runner [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints +in 15.0. + +Instead of the GitLab Runner API endpoints returning `offline` and `not_connected` for runners that have not +contacted the GitLab instance in the past three months, the API endpoints now return the `stale` value, +which was introduced in 14.6. + +### Audit events for repository push events + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are removed as of GitLab 15.0. + +Audit events for repository events were always disabled by default and had to be manually enabled with a feature flag. +Enabling them could slow down GitLab instances by generating too many events. Therefore, they are removed. + +Please note that we will add high-volume audit events in the future as part of [streaming audit events](https://docs.gitlab.com/ee/administration/audit_event_streaming.html). An example of this is how we will send [Git fetch actions](https://gitlab.com/gitlab-org/gitlab/-/issues/343984) as a streaming audit event. If you would be interested in seeing repository push events or some other action as a streaming audit event, please reach out to us! + +### Background upload for object storage + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` has been removed in GitLab 15.0. + +This impacts a small subset of object storage providers, including but not limited to: + +- **OpenStack** Customers using OpenStack need to change their configuration to use the S3 API instead of Swift. +- **RackSpace** Customers using RackSpace-based object storage need to migrate data to a different provider. + +If your object storage provider does not support `background_upload`, please [migrate objects to a supported object storage provider](https://docs.gitlab.com/ee/administration/object_storage.html#migrate-objects-to-a-different-object-storage-provider). + +### Container Network and Host Security + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). + +As part of this change, the following capabilities within GitLab are scheduled for removal in GitLab 15.0: + +- The **Security & Compliance > Threat Monitoring** page. +- The Network Policy security policy type, as found on the **Security & Compliance > Policies** page. +- The ability to manage integrations with the following technologies through GitLab: AppArmor, Cilium, Falco, FluentD, and Pod Security Policies. +- All APIs related to the above functionality. + +For additional context, or to provide feedback regarding this change, please reference our [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7476). + +### Container registry authentication with htpasswd + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. + +Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. + +### Custom `geo:db:*` Rake tasks are no longer available + +In GitLab 14.8, we [deprecated the `geo:db:*` Rake tasks and replaced them with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458). +The following `geo:db:*` tasks have been removed from GitLab 15.0 and have been replaced with their corresponding `db:*:geo` tasks: + +- `geo:db:drop` -> `db:drop:geo` +- `geo:db:create` -> `db:create:geo` +- `geo:db:setup` -> `db:setup:geo` +- `geo:db:migrate` -> `db:migrate:geo` +- `geo:db:rollback` -> `db:rollback:geo` +- `geo:db:version` -> `db:version:geo` +- `geo:db:reset` -> `db:reset:geo` +- `geo:db:seed` -> `db:seed:geo` +- `geo:schema:load:geo` -> `db:schema:load:geo` +- `geo:db:schema:dump` -> `db:schema:dump:geo` +- `geo:db:migrate:up` -> `db:migrate:up:geo` +- `geo:db:migrate:down` -> `db:migrate:down:geo` +- `geo:db:migrate:redo` -> `db:migrate:redo:geo` +- `geo:db:migrate:status` -> `db:migrate:status:geo` +- `geo:db:test:prepare` -> `db:test:prepare:geo` +- `geo:db:test:load` -> `db:test:load:geo` +- `geo:db:test:purge` -> `db:test:purge:geo` + +### DS_DEFAULT_ANALYZERS environment variable + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +We are removing the `DS_DEFAULT_ANALYZERS` environment variable from Dependency Scanning on May 22, 2022 in 15.0. After this removal, this variable's value will be ignored. To configure which analyzers to run with the default configuration, you should use the `DS_EXCLUDED_ANALYZERS` variable instead. + +### Dependency Scanning default Java version changed to 17 + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +For Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency Scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_JAVA_VERSION` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). Please note that consequently the default version of Gradle is now 7.3.3. + +### ELK stack logging removed in GitLab 15.0 + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users could search for relevant logs in GitLab directly. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, this feature is no longer available. For more information on the future of logging and observability, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). + +### Elasticsearch 6.8.x in GitLab 15.0 + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Elasticsearch 6.8 support has been removed in GitLab 15.0. Elasticsearch 6.8 has reached [end of life](https://www.elastic.co/support/eol). +If you use Elasticsearch 6.8, **you must upgrade your Elasticsearch version to 7.x** prior to upgrading to GitLab 15.0. +You should not upgrade to Elasticsearch 8 until you have completed the GitLab 15.0 upgrade. + +View the [version requirements](https://docs.gitlab.com/ee/integration/elasticsearch.html#version-requirements) for details. + +### End of support for Python 3.6 in Dependency Scanning + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +For those using Dependency Scanning for Python projects, we are removing support for the default `gemnasium-python:2` image which uses Python 3.6, as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). + +### External status check API breaking changes + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to +support pass-by-default requests to mark a status check as passing. Pass-by-default requests are now removed. +Specifically, the following are removed: + +- Requests that do not contain the `status` field. +- Requests that have the `status` field set to `approved`. + +From GitLab 15.0, status checks are only set to a passing state if the `status` field is both present +and set to `passed`. Requests that: + +- Do not contain the `status` field will be rejected with a `400` error. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338827). +- Contain any value other than `passed`, such as `approved`, cause the status check to fail. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/339039). + +To align with this change, API calls to list external status checks also return the value of `passed` rather than +`approved` for status checks that have passed. + +### GitLab Serverless + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +All functionality related to GitLab Serverless was deprecated in GitLab 14.3 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to explore using the following technologies with GitLab CI/CD: + +- [Serverless Framework](https://www.serverless.com) +- [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/deploying-using-gitlab.html) + +For additional context, or to provide feedback regarding this change, please reference our [deprecation issue](https://gitlab.com/groups/gitlab-org/configure/-/epics/6). + +### Gitaly nodes in virtual storage + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Configuring the Gitaly nodes directly in the virtual storage's root configuration object has been deprecated in GitLab 13.12 and is no longer supported in GitLab 15.0. You must move the Gitaly nodes under the `'nodes'` key as described in [the Praefect configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#praefect). + +### GraphQL permissions change for Package settings + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. + +The permissions model for GraphQL is being updated. After 15.0, users with the Guest, Reporter, and Developer role can no longer update these settings: + +- [Package Registry settings](https://docs.gitlab.com/ee/api/graphql/reference/#packagesettings) +- [Container Registry cleanup policy](https://docs.gitlab.com/ee/api/graphql/reference/#containerexpirationpolicy) +- [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) +- [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting) + +The issue for this removal is [GitLab-#350682](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) + +### Jaeger integration removed in GitLab 15.0 + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users could previously navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab was deprecated in GitLab 14.7, and removed in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). + +### Known host required for GitLab Runner SSH executor + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml`. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. + +In GitLab 15.0, the default value for this configuration option has changed from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. + +### Legacy Geo Admin UI routes + +In GitLab 13.0, we introduced new project and design replication details routes in the Geo Admin UI. These routes are `/admin/geo/replication/projects` and `/admin/geo/replication/designs`. We kept the legacy routes and redirected them to the new routes. These legacy routes `/admin/geo/projects` and `/admin/geo/designs` have been removed in GitLab 15.0. Please update any bookmarks or scripts that may use the legacy routes. + +### Legacy approval status names in License Compliance API + +We have now removed the deprecated legacy names for approval status of license policy (`blacklisted`, `approved`) in the API queries and responses. If you are using our License Compliance API you should stop using the `approved` and `blacklisted` query parameters, they are now `allowed` and `denied`. In 15.0 the responses will also stop using `approved` and `blacklisted` so you may need to adjust any of your custom tools. + +### Move Gitaly Cluster Praefect `database_host_no_proxy` and `database_port_no_proxy configs` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The Gitaly Cluster configuration keys for `praefect['database_host_no_proxy']` and `praefect['database_port_no_proxy']` are replaced with `praefect['database_direct_host']` and `praefect['database_direct_port']`. + +### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and is removed from GitLab Shell in GitLab 15.0. + +### OAuth implicit grant + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The OAuth implicit grant authorization flow is no longer supported. Any applications that use OAuth implicit grant must switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). + +### OAuth tokens without an expiration + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +GitLab no longer supports OAuth tokens [without an expiration](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens). + +Any existing token without an expiration has one automatically generated and applied. + +### Optional enforcement of SSH expiration + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Disabling SSH expiration enforcement is unusual from a security perspective and could create unusual situations where an expired +key is unintentionally able to be used. Unexpected behavior in a security feature is inherently dangerous and so now we enforce +expiration on all SSH keys. + +### Optional enforcement of personal access token expiration + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Allowing expired personal access tokens to be used is unusual from a security perspective and could create unusual situations where an +expired key is unintentionally able to be used. Unexpected behavior in a security feature is inherently dangerous and so we now do not let expired personal access tokens be used. + +### Out-of-the-box SAST (SpotBugs) support for Java 8 + +The [GitLab SAST SpotBugs analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) scans [Java, Scala, Groovy, and Kotlin code](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) for security vulnerabilities. +For technical reasons, the analyzer must first compile the code before scanning. +Unless you use the [pre-compilation strategy](https://docs.gitlab.com/ee/user/application_security/sast/#pre-compilation), the analyzer attempts to automatically compile your project's code. + +In GitLab versions prior to 15.0, the analyzer image included Java 8 and Java 11 runtimes to facilitate compilation. + +As of GitLab 15.0, we've: + +- Removed Java 8 from the analyzer image to reduce the size of the image. +- Added Java 17 to the analyzer image to make it easier to compile with Java 17. +- Changed the default Java version from Java 8 to Java 17. + +If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change). + +### Pseudonymizer + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The Pseudonymizer feature is generally unused, can cause production issues with large databases, and can interfere with object storage development. +It was removed in GitLab 15.0. + +### Remove Versions from PackageType + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). + +In GitLab 15.0, we will completely remove `Version` from `PackageType`. + +### Remove `promote-to-primary-node` command from `gitlab-ctl` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` has been removed in GitLab 15.0. + +### Remove `type` and `types` keyword from CI/CD configuration + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The `type` and `types` CI/CD keywords is removed in GitLab 15.0, so pipelines that use these keywords fail with a syntax error. Switch to `stage` and `stages`, which have the same behavior. + +### Remove dependency_proxy_for_private_groups feature flag + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +A feature flag was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 as part of the change to require authentication to use the Dependency Proxy. Before GitLab 13.7, you could use the Dependency Proxy without authentication. + +In GitLab 15.0, we will remove the feature flag, and you must always authenticate when you use the Dependency Proxy. + +### Remove pipelines field from the version field + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: + +- The `versions` field's `pipelines` field. This returns all the pipelines associated with all the package's versions, which can pull an unbounded number of objects in memory and create performance concerns. +- The `pipelines` field of a specific `version`. This returns only the pipelines associated with that single package version. + +To mitigate possible performance problems, we will remove the `versions` field's `pipelines` field in GitLab 15.0. Although you will no longer be able to get all pipelines for all versions of a package, you can still get the pipelines of a single version through the remaining `pipelines` field for that version. + +### Removed feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` has been removed in GitLab 15.0. From now on, push rules will supersede CODEOWNERS. The CODEOWNERS feature is no longer available for access control. + +### Request profiling + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +[Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/request_profiling.html) has been removed in GitLab 15.0. + +We're working on [consolidating our profiling tools](https://gitlab.com/groups/gitlab-org/-/epics/7327) and making them more easily accessible. +We [evaluated](https://gitlab.com/gitlab-org/gitlab/-/issues/350152) the use of this feature and we found that it is not widely used. +It also depends on a few third-party gems that are not actively maintained anymore, have not been updated for the latest version of Ruby, or crash frequently when profiling heavy page loads. + +For more information, check the [summary section of the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488#deprecation-summary). + +### Required pipeline configurations in Premium tier + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +[Required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) helps to define and mandate organization-wide pipeline configurations and is a requirement at an executive and organizational level. To align better with our [pricing philosophy](https://about.gitlab.com/company/pricing/#three-tiers), this feature is removed from the Premium tier in GitLab 15.0. This feature continues to be available in the GitLab Ultimate tier. + +We recommend customers use [Compliance Pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration), also in GitLab Ultimate, as an alternative as it provides greater flexibility, allowing required pipelines to be assigned to specific compliance framework labels. + +This change also helps GitLab remain consistent in our tiering strategy with the other related Ultimate-tier features: + +- [Security policies](https://docs.gitlab.com/ee/user/application_security/policies/). +- [Compliance framework pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration). + +### Retire-JS Dependency Scanning tool + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +We have removed support for retire.js from Dependency Scanning as of May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. + +If you have explicitly excluded retire.js using the `DS_EXCLUDED_ANALYZERS` variable, then you will be able to remove the reference to retire.js. If you have customized your pipeline’s Dependency Scanning configuration related to the `retire-js-dependency_scanning` job, then you will want to switch to `gemnasium-dependency_scanning`. If you have not used the `DS_EXCLUDED_ANALYZERS` to reference retire.js, or customized your template specifically for retire.js, you will not need to take any action. + +### Runner status `not_connected` API value + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The GitLab Runner REST and GraphQL [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints +deprecated the `not_connected` status value in GitLab 14.6 and will start returning `never_contacted` in its place +starting in GitLab 15.0. + +Runners that have never contacted the GitLab instance will also return `stale` if created more than 3 months ago. + +### SAST support for .NET 2.1 + +The [GitLab SAST Security Code Scan analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) scans .NET code for security vulnerabilities. +For technical reasons, the analyzer must first build the code to scan it. + +In GitLab versions prior to 15.0, the default analyzer image (version 2) included support for: + +- .NET 2.1 +- .NET Core 3.1 +- .NET 5.0 + +In GitLab 15.0, we've changed the default major version for this analyzer from version 2 to version 3. This change: + +- Adds [severity values for vulnerabilities](https://gitlab.com/gitlab-org/gitlab/-/issues/350408) along with [other new features and improvements](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan/-/blob/master/CHANGELOG.md). +- Removes .NET 2.1 support. +- Adds support for .NET 6.0, Visual Studio 2019, and Visual Studio 2022. + +Version 3 was [announced in GitLab 14.6](https://about.gitlab.com/releases/2021/12/22/gitlab-14-6-released/#sast-support-for-net-6) and made available as an optional upgrade. + +If you rely on .NET 2.1 support being present in the analyzer image by default, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352553#breaking-change). + +### SUSE Linux Enterprise Server 12 SP2 + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. + +### Secret Detection configuration variables + +To make it simpler and more reliable to [customize GitLab Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings), we've removed some of the variables that you could previously set in your CI/CD configuration. + +The following variables previously allowed you to customize the options for historical scanning, but interacted poorly with the [GitLab-managed CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) and are now removed: + +- `SECRET_DETECTION_COMMIT_FROM` +- `SECRET_DETECTION_COMMIT_TO` +- `SECRET_DETECTION_COMMITS` +- `SECRET_DETECTION_COMMITS_FILE` + +The `SECRET_DETECTION_ENTROPY_LEVEL` previously allowed you to configure rules that only considered the entropy level of strings in your codebase, and is now removed. +This type of entropy-only rule created an unacceptable number of incorrect results (false positives). + +You can still customize the behavior of the Secret Detection analyzer using the [available CI/CD variables](https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-cicd-variables). + +For further details, see [the deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352565). + +### Sidekiq configuration for metrics and health checks + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GitLab 15.0, you can no longer serve Sidekiq metrics and health checks over a single address and port. + +To improve stability, availability, and prevent data loss in edge cases, GitLab now serves +[Sidekiq metrics and health checks from two separate servers](https://gitlab.com/groups/gitlab-org/-/epics/6409). + +When you use Omnibus or Helm charts, if GitLab is configured for both servers to bind to the same address, +a configuration error occurs. +To prevent this error, choose different ports for the metrics and health check servers: + +- [Configure Sidekiq health checks](https://docs.gitlab.com/ee/administration/sidekiq.html#configure-health-checks) +- [Configure the Sidekiq metrics server](https://docs.gitlab.com/ee/administration/sidekiq.html#configure-the-sidekiq-metrics-server) + +If you installed GitLab from source, verify manually that both servers are configured to bind to separate addresses and ports. + +### Static Site Editor + +The Static Site Editor was deprecated in GitLab 14.7 and the feature is being removed in GitLab 15.0. Incoming requests to the Static Site Editor will be redirected and open the target file to edit in the Web IDE. Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/static_site_editor/) for more information, including how to remove the configuration files from existing projects. We will continue investing in improvements to the Markdown editing experience by [maturing the Content Editor](https://gitlab.com/groups/gitlab-org/-/epics/5401) and making it available as a way to edit content across GitLab. + +### Support for `gitaly['internal_socket_dir']` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +Gitaly introduced a new directory that holds all runtime data Gitaly requires to operate correctly. This new directory replaces the old internal socket directory, and consequentially the usage of `gitaly['internal_socket_dir']` was deprecated in favor of `gitaly['runtime_dir']`. + +### Support for legacy format of `config/database.yml` removed + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The syntax of [GitLab's database](https://docs.gitlab.com/omnibus/settings/database.html) +configuration located in `database.yml` has changed and the legacy format has been removed. +The legacy format supported a single PostgreSQL adapter, whereas the new format supports multiple databases. +The `main:` database needs to be defined as a first configuration item. + +This change only impacts users compiling GitLab from source, all the other installation methods handle this configuration automatically. +Instructions are available [in the source update documentation](https://docs.gitlab.com/ee/update/upgrading_from_source.html#new-configuration-options-for-databaseyml). + +### Test coverage project CI/CD setting + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +To specify a test coverage pattern, in GitLab 15.0 the +[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-removed) +has been removed. + +To set test coverage parsing, use the project’s `.gitlab-ci.yml` file by providing a regular expression with the +[`coverage` keyword](https://docs.gitlab.com/ee/ci/yaml/index.html#coverage). + +### The `promote-db` command is no longer available from `gitlab-ctl` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. The `gitlab-ctl promote-db` command has been removed in GitLab 15.0. + +### Update to the Container Registry group-level API + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +In GitLab 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). + +The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. + +### Vulnerability Check + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The vulnerability check feature was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. + +The new security approvals feature is similar to vulnerability check. For example, both can require approvals for MRs that contain security vulnerabilities. However, security approvals improve the previous experience in several ways: + +- Users can choose who is allowed to edit security approval rules. An independent security or compliance team can therefore manage rules in a way that prevents development project maintainers from modifying the rules. +- Multiple rules can be created and chained together to allow for filtering on different severity thresholds for each scanner type. +- A two-step approval process can be enforced for any desired changes to security approval rules. +- A single set of security policies can be applied to multiple development projects to allow for ease in maintaining a single, centralized ruleset. + +### `Managed-Cluster-Applications.gitlab-ci.yml` + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The `Managed-Cluster-Applications.gitlab-ci.yml` CI/CD template is being removed. If you need an alternative, try the [Cluster Management project template](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) instead. If your are not ready to move, you can copy the [last released version](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/v14.10.1/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml) of the template into your project. + +### `artifacts:report:cobertura` keyword + +As of GitLab 15.0, the [`artifacts:report:cobertura`](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscobertura-removed) +keyword has been [replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/344533) by +[`artifacts:reports:coverage_report`](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report). +Cobertura is the only supported report file, but this is the first step towards GitLab supporting other report types. + +### `omniauth-kerberos` gem + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The `omniauth-kerberos` gem is no longer supported. This gem has not been maintained and has very little usage. Therefore, we +removed support for this authentication method and recommend using [SPEGNO](https://en.wikipedia.org/wiki/SPNEGO) instead. You can +follow the [upgrade instructions](https://docs.gitlab.com/ee/integration/kerberos.html#upgrading-from-password-based-to-ticket-based-kerberos-sign-ins) +to upgrade from the removed integration to the new supported one. + +We are not removing Kerberos SPNEGO integration. We are removing the old password-based Kerberos. + +### bundler-audit Dependency Scanning tool + +WARNING: +This feature was changed or removed in 15.0 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal, Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. + +If you have explicitly excluded bundler-audit using the `DS_EXCLUDED_ANALYZERS` variable, then you will be able to remove the reference to bundler-audit. If you have customized your pipeline’s Dependency Scanning configuration related to the `bundler-audit-dependency_scanning` job, then you will want to switch to `gemnasium-dependency_scanning`. If you have not used the `DS_EXCLUDED_ANALYZERS` to reference bundler-audit or customized your template specifically for bundler-audit, you will not need to take any action. + +## 14.10 + +### Permissions change for downloading Composer dependencies + +WARNING: +This feature was changed or removed in 14.10 +as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Before updating GitLab, review the details carefully to determine if you need to make any +changes to your code, settings, or workflow. + +The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. + +Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. + ## 14.9 ### Integrated error tracking disabled by default @@ -496,9 +1184,9 @@ as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#brea Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -PostgreSQL 12 will be the minimum required version in GitLab 14.0. It offers [significant improvements](https://www.postgresql.org/about/news/postgresql-12-released-1976/) to indexing, partitioning, and general performance benefits. +GitLab 14.0 requires PostgreSQL 12 or later. It offers [significant improvements](https://www.postgresql.org/about/news/postgresql-12-released-1976/) to indexing, partitioning, and general performance benefits. -Starting in GitLab 13.7, all new installations default to version 12. From GitLab 13.8, single-node instances are automatically upgraded as well. If you aren't ready to upgrade, you can [opt out of automatic upgrades](https://docs.gitlab.com/omnibus/settings/database.html#opt-out-of-automatic-postgresql-upgrades). +Starting in GitLab 13.7, all new installations default to PostgreSQL version 12. From GitLab 13.8, single-node instances are automatically upgraded as well. If you aren't ready to upgrade, you can [opt out of automatic upgrades](https://docs.gitlab.com/omnibus/settings/database.html#opt-out-of-automatic-postgresql-upgrades). ### Redundant timestamp field from DORA metrics API payload diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 4537faaaead..485dbc1b0bc 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -7,10 +7,6 @@ comments: false # Upgrading Community Edition and Enterprise Edition from source **(FREE SELF)** -NOTE: -Users wishing to upgrade to 12.0.0 must take some extra steps. See the -version specific upgrade instructions for 12.0.0 for more details. - Make sure you view this update guide from the branch (version) of GitLab you would like to install (for example, `11.8`). You can select the required version of documentation in the dropdown at the top right corner of GitLab documentation page. @@ -73,9 +69,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.4.tar.gz" -echo '3043099089608859fc8cce7f9fdccaa1f53a462457e3838ec3b25a7d609fbc5b ruby-2.7.4.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.4.tar.gz -cd ruby-2.7.4 +curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz" +echo '2755b900a21235b443bb16dadd9032f784d4a88f143d852bc5d154f22b8781f1 ruby-2.7.5.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.5.tar.gz +cd ruby-2.7.5 ./configure --disable-install-rdoc --enable-shared make @@ -150,7 +146,7 @@ Remember to set `git -> bin_path` to `/usr/local/bin/git` in `config/gitlab.yml` ### 7. Update PostgreSQL WARNING: -From GitLab 14.0, you must use at least PostgreSQL 12. +GitLab 14.0 requires at least PostgreSQL 12. The latest version of GitLab might depend on a more recent PostgreSQL version than what you are running. You may also have to enable some @@ -421,6 +417,39 @@ Example: Additional instructions here. --> +### 15.0.0 + +Support for more than one database has been added to GitLab. [As part of this](https://gitlab.com/gitlab-org/gitlab/-/issues/338182), +`config/database.yml` needs to include a database name in the database configuration. +The `main: database` must be first. If an invalid or deprecated syntax is used, an error is generated +during application start: + +```plaintext +ERROR: This installation of GitLab uses unsupported 'config/database.yml'. +The main: database needs to be defined as a first configuration item instead of primary. (RuntimeError) +``` + +Previously, the `config/database.yml` file looked like the following: + +```yaml +production: + adapter: postgresql + encoding: unicode + database: gitlabhq_production + ... +``` + +Starting with GitLab 15.0, it needs to define a `main` database first: + +```yaml +production: + main: + adapter: postgresql + encoding: unicode + database: gitlabhq_production + ... +``` + ### 14.5.0 As part of [enabling real-time issue assignees](https://gitlab.com/gitlab-org/gitlab/-/issues/330117), Action Cable is now enabled by default, and requires `config/cable.yml` to be present. diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index 5aa80c12f11..99457ef98d6 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -557,7 +557,7 @@ On each **secondary** node, executing the following: 1. Run post-deployment database migrations, specific to the Geo database ```shell - sudo gitlab-rake geo:db:migrate + sudo gitlab-rake db:migrate:geo ``` After all **secondary** nodes are updated, finalize @@ -790,7 +790,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure ```shell sudo gitlab-ctl reconfigure - sudo SKIP_POST_DEPLOYMENT_MIGRATIONS=true gitlab-rake geo:db:migrate + sudo SKIP_POST_DEPLOYMENT_MIGRATIONS=true gitlab-rake db:migrate:geo ``` 1. If this deploy node is normally used to serve requests or perform @@ -866,7 +866,7 @@ sudo gitlab-ctl restart geo-logcursor 1. Run post-deployment database migrations, specific to the Geo database: ```shell - sudo gitlab-rake geo:db:migrate + sudo gitlab-rake db:migrate:geo ``` 1. Verify Geo configuration and dependencies diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md index 69cb2f04c4d..9d6dcf30908 100644 --- a/doc/user/admin_area/broadcast_messages.md +++ b/doc/user/admin_area/broadcast_messages.md @@ -20,7 +20,7 @@ Broadcast messages can be managed using the [broadcast messages API](../../api/b Banners are shown on the top of a page and in Git remote responses. - + ```shell $ git push @@ -66,18 +66,13 @@ To add a broadcast message: - `padding` - `margin` - `text-decoration` -1. Select one of the suggested background colors, or add the hex code of a different color. The default color is orange. +1. Select a **Theme**. The default theme is `indigo`. 1. Select the **Dismissable** checkbox to enable users to dismiss the broadcast message. 1. Optional. Select **Target roles** to only show the broadcast message to users with the selected roles. The message displays on group, subgroup, and project pages, and does not display in Git remote responses. 1. If required, add a **Target Path** to only show the broadcast message on URLs matching that path. You can use the wildcard character `*` to match multiple URLs, for example `mygroup/myproject*`. 1. Select a date for the message to start and end. 1. Select **Add broadcast message**. -NOTE: -The **Background color** field expects the value to be a hexadecimal code because -the form uses the [color_field](https://api.rubyonrails.org/v6.0.3.4/classes/ActionView/Helpers/FormHelper.html#method-i-color_field) -helper method, which generates the proper HTML to render. - When a broadcast message expires, it no longer displays in the user interface but is still listed in the list of broadcast messages. diff --git a/doc/user/admin_area/credentials_inventory.md b/doc/user/admin_area/credentials_inventory.md index 21ac0f720ec..4308b45df78 100644 --- a/doc/user/admin_area/credentials_inventory.md +++ b/doc/user/admin_area/credentials_inventory.md @@ -40,14 +40,11 @@ To access the Credentials inventory: If you see a **Revoke** button, you can revoke that user's PAT. Whether you see a **Revoke** button depends on the token state, and if an expiration date has been set. For more information, see the following table: -| Token state | [Token expiration enforced?](settings/account_and_limit_settings.md#allow-expired-personal-access-tokens-to-be-used-deprecated) | Show Revoke button? | Comments | -|-------------|------------------------|--------------------|----------------------------------------------------------------------------| -| Active | Yes | Yes | Allows administrators to revoke the PAT, such as for a compromised account | -| Active | No | Yes | Allows administrators to revoke the PAT, such as for a compromised account | -| Expired | Yes | No | PAT expires automatically | -| Expired | No | Yes | The administrator may revoke the PAT to prevent indefinite use | -| Revoked | Yes | No | Not applicable; token is already revoked | -| Revoked | No | No | Not applicable; token is already revoked | +| Token state | Show Revoke button? | Comments | +|-------------|---------------------|----------------------------------------------------------------------------| +| Active | Yes | Allows administrators to revoke the PAT, such as for a compromised account | +| Expired | No | Not applicable; token is already expired | +| Revoked | No | Not applicable; token is already revoked | When a PAT is revoked from the credentials inventory, the instance notifies the user by email. diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index b3b2c14adbd..43dce1921f4 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -12,7 +12,7 @@ You can configure various settings for GitLab Geo sites. For more information, s On either the primary or secondary site: 1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Geo > Nodes**. +1. On the left sidebar, select **Geo > Sites**. ## Common settings diff --git a/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png b/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png Binary files differdeleted file mode 100644 index 2e893476bc6..00000000000 --- a/doc/user/admin_area/img/broadcast_messages_banner_v12_10.png +++ /dev/null diff --git a/doc/user/admin_area/img/broadcast_messages_banner_v15_0.png b/doc/user/admin_area/img/broadcast_messages_banner_v15_0.png Binary files differnew file mode 100644 index 00000000000..e1b350142b3 --- /dev/null +++ b/doc/user/admin_area/img/broadcast_messages_banner_v15_0.png diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index f57672d3d36..262bb2cc931 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -25,7 +25,7 @@ The Admin Area is made up of the following sections: | Section | Description | |:-----------------------------------------------|:------------| | **{overview}** [Overview](#overview-section) | View your GitLab [Dashboard](#admin-area-dashboard), and administer [projects](#administering-projects), [users](#administering-users), [groups](#administering-groups), [topics](#administering-topics), [jobs](#administering-jobs), [runners](#administering-runners), and [Gitaly servers](#administering-gitaly-servers). | -| **{monitor}** Monitoring | View GitLab [system information](#system-information), and information on [background jobs](#background-jobs), [logs](#logs), [health checks](monitoring/health_check.md), [requests profiles](#requests-profiles), and [audit events](#audit-events). | +| **{monitor}** Monitoring | View GitLab [system information](#system-information), and information on [background jobs](#background-jobs), [logs](#logs), [health checks](monitoring/health_check.md), and [audit events](#audit-events). | | **{messages}** Messages | Send and manage [broadcast messages](broadcast_messages.md) for your users. | | **{hook}** System Hooks | Configure [system hooks](../../administration/system_hooks.md) for many events. | | **{applications}** Applications | Create system [OAuth applications](../../integration/oauth_provider.md) for integrations with other services. | @@ -184,7 +184,7 @@ The following data is included in the export: - Type - Path - Access level ([Project](../permissions.md#project-members-permissions) and [Group](../permissions.md#group-members-permissions)) -- Date of last activity ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345388) in GitLab 14.6). For a list of activities that populate this column, see the [Users API documentation](../../api/users.md#get-user-activities-admin-only). +- Date of last activity ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345388) in GitLab 14.6). For a list of activities that populate this column, see the [Users API documentation](../../api/users.md#get-user-activities-administrator-only). Only the first 100,000 user accounts are exported. @@ -326,7 +326,7 @@ To search runners' descriptions: 1. In the **Search or filter results...** field, type the description of the runner you want to find. -1. Press Enter. +1. Press <kbd>Enter</kbd>. You can also filter runners by status, type, and tag. To filter: @@ -430,10 +430,6 @@ For details of these log files and their contents, see [Log system](../../admini The content of each log file is listed in chronological order. To minimize performance issues, a maximum 2000 lines of each log file are shown. -### Requests Profiles - -The **Requests Profiles** page contains the token required for profiling. For more details, see [Request Profiling](../../administration/monitoring/performance/request_profiling.md). - ### Audit Events **(PREMIUM SELF)** The **Audit Events** page lists changes made within the GitLab server. With this information you can control, analyze, and track every change. diff --git a/doc/user/admin_area/license_file.md b/doc/user/admin_area/license_file.md index 5999e774d26..ff9e87680f9 100644 --- a/doc/user/admin_area/license_file.md +++ b/doc/user/admin_area/license_file.md @@ -18,8 +18,7 @@ Otherwise, to add your license: 1. Sign in to GitLab as an administrator. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. -1. In the **License file** area, select **Add a license**. -1. Add a license by either uploading the file or pasting the key. +1. In the **Add License** area, add a license by either uploading the file or entering the key. 1. Select the **Terms of Service** checkbox. 1. Select **Add license**. diff --git a/doc/user/admin_area/moderate_users.md b/doc/user/admin_area/moderate_users.md index e8db319df77..53c08d8cbc1 100644 --- a/doc/user/admin_area/moderate_users.md +++ b/doc/user/admin_area/moderate_users.md @@ -15,7 +15,7 @@ users. A user in _pending approval_ state requires action by an administrator. A user sign up can be in a pending approval state because an administrator has enabled any of the following options: -- [Require admin approval for new sign-ups](settings/sign_up_restrictions.md#require-administrator-approval-for-new-sign-ups) setting. +- [Require administrator approval for new sign-ups](settings/sign_up_restrictions.md#require-administrator-approval-for-new-sign-ups) setting. - [User cap](settings/sign_up_restrictions.md#user-cap). - [Block auto-created users (OmniAuth)](../../integration/omniauth.md#configure-initial-settings) - [Block auto-created users (LDAP)](../../administration/auth/ldap/index.md#basic-configuration-settings) diff --git a/doc/user/admin_area/monitoring/background_migrations.md b/doc/user/admin_area/monitoring/background_migrations.md index 726827054da..b666c0c5ad2 100644 --- a/doc/user/admin_area/monitoring/background_migrations.md +++ b/doc/user/admin_area/monitoring/background_migrations.md @@ -190,7 +190,7 @@ sudo gitlab-rake gitlab:background_migrations:finalize[CopyColumnUsingBackground In GitLab 14.8, the `BackfillNamespaceIdForNamespaceRoute` batched background migration job may fail to complete. When retried, a `500 Server Error` is returned. This issue was -[resolved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82387) in GitLab 14.9. +[resolved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82387) in GitLab 14.9. To resolve this issue, [upgrade GitLab](../../../update/index.md) from 14.8 to 14.9. You can ignore the failed batch migration until after you update to GitLab 14.9. diff --git a/doc/user/admin_area/reporting/spamcheck.md b/doc/user/admin_area/reporting/spamcheck.md index 559235fe322..b1ec203cffc 100644 --- a/doc/user/admin_area/reporting/spamcheck.md +++ b/doc/user/admin_area/reporting/spamcheck.md @@ -65,4 +65,4 @@ Spamcheck service on its own can not communicate directly over TLS with GitLab. However, Spamcheck can be deployed behind a reverse proxy which performs TLS termination. In such a scenario, GitLab can be made to communicate with Spamcheck over TLS by specifying `tls://` scheme for the external Spamcheck URL -instead of `grpc://` in the Admin settings. +instead of `grpc://` in the Admin Area settings. diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index e6d8107ed9b..9905298784a 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -51,12 +51,14 @@ For GitLab.com repository size limits, read [accounts and limit settings](../../ ## Max push size -You can change the maximum push size for your repository: +You can change the maximum push size for your instance: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**, then expand **Account and limit**. 1. Increase or decrease by changing the value in **Maximum push size (MB)**. +For GitLab.com application limits, read [GitLab application limits](../../../administration/instance_limits.md#max-push-size). + NOTE: When you [add files to a repository](../../project/repository/web_editor.md#create-a-file) through the web UI, the maximum **attachment** size is the limiting factor, @@ -64,9 +66,19 @@ because the [web server](../../../development/architecture.md#components) must receive the file before GitLab can generate the commit. Use [Git LFS](../../../topics/git/lfs/index.md) to add large files to a repository. +## Max export size + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86124) in GitLab 15.0. + +To modify the maximum file size for exports in GitLab: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > General**, then expand **Account and limit**. +1. Increase or decrease by changing the value in **Maximum export size (MB)**. + ## Max import size -> [Modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to unlimited in GitLab 13.8. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to unlimited in GitLab 13.8. To modify the maximum file size for imports in GitLab: @@ -158,22 +170,6 @@ wiki, packages, or snippets. The repository size limit applies to both private a For details on manually purging files, see [reducing the repository size using Git](../../project/repository/reducing_the_repo_size_using_git.md). -## Troubleshooting - -### 413 Request Entity Too Large - -When attaching a file to a comment or reply in GitLab displays a `413 Request Entity Too Large` -error, the [max attachment size](#max-attachment-size) -is probably larger than the web server's allowed value. - -To increase the max attachment size to 200 MB in a -[Omnibus GitLab](https://docs.gitlab.com/omnibus/) install, you may need to -add the line below to `/etc/gitlab/gitlab.rb` before increasing the max attachment size: - -```ruby -nginx['client_max_body_size'] = "200m" -``` - ## Customize session duration for Git Operations when 2FA is enabled **(PREMIUM SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296669) in GitLab 13.9. @@ -229,35 +225,28 @@ Once a lifetime for SSH keys is set, GitLab: NOTE: When a user's SSH key becomes invalid they can delete and re-add the same key again. -## Allow expired SSH keys to be used (DEPRECATED) **(ULTIMATE SELF)** +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> +## Allow expired SSH keys to be used (removed) **(ULTIMATE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250480) in GitLab 13.9. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/320970) in GitLab 14.0. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 15.0. -WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8. +This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 15.0. +<!--- end_remove --> -By default, expired SSH keys **are not usable**. - -To allow the use of expired SSH keys: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > General**. -1. Expand the **Account and limit** section. -1. Uncheck the **Enforce SSH key expiration** checkbox. - -Disabling SSH key expiration immediately enables all expired SSH keys. - -## Limit the lifetime of personal access tokens **(ULTIMATE SELF)** +## Limit the lifetime of access tokens **(ULTIMATE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3649) in GitLab 12.6. Users can optionally specify a lifetime for -[personal access tokens](../../profile/personal_access_tokens.md). +access tokens, this includes [personal](../../profile/personal_access_tokens.md), +[group](../../group/settings/group_access_tokens.md), and [project](../../project/settings/project_access_tokens.md) access tokens. This lifetime is not a requirement, and can be set to any arbitrary number of days. -Personal access tokens are the only tokens needed for programmatic access to GitLab. +Access tokens are the only tokens needed for programmatic access to GitLab. However, organizations with security requirements may want to enforce more protection by requiring the regular rotation of these tokens. @@ -266,15 +255,15 @@ requiring the regular rotation of these tokens. Only a GitLab administrator can set a lifetime. Leaving it empty means there are no restrictions. -To set a lifetime on how long personal access tokens are valid: +To set a lifetime on how long access tokens are valid: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. -1. Fill in the **Maximum allowable lifetime for personal access tokens (days)** field. +1. Fill in the **Maximum allowable lifetime for access tokens (days)** field. 1. Click **Save changes**. -Once a lifetime for personal access tokens is set, GitLab: +Once a lifetime for access tokens is set, GitLab: - Applies the lifetime for new personal access tokens, and require users to set an expiration date and a date no later than the allowed lifetime. @@ -282,23 +271,17 @@ Once a lifetime for personal access tokens is set, GitLab: allowed lifetime. Three hours is given to allow administrators to change the allowed lifetime, or remove it, before revocation takes place. -## Allow expired Personal Access Tokens to be used (DEPRECATED) **(ULTIMATE SELF)** +<!-- start_remove The following content will be removed on remove_date: '2022-08-22' --> +## Allow expired access tokens to be used (removed) **(ULTIMATE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214723) in GitLab 13.1. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/296881) in GitLab 13.9. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351962) in GitLab 14.8. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351962) in GitLab 15.0. -WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351962) in GitLab 14.8. - -By default, expired personal access tokens (PATs) **are not usable**. - -To allow the use of expired PATs: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > General**. -1. Expand the **Account and limit** section. -1. Uncheck the **Enforce personal access token expiration** checkbox. +This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351962) in GitLab 15.0. +<!-- end_remove --> ## Disable user profile name changes **(PREMIUM SELF)** @@ -314,5 +297,35 @@ To do this: NOTE: When this ability is disabled, GitLab administrators can still use the -[Admin UI](../index.md#administering-users) or the +[Admin Area](../index.md#administering-users) or the [API](../../../api/users.md#user-modification) to update usernames. + +## Troubleshooting + +### 413 Request Entity Too Large + +When attaching a file to a comment or reply in GitLab displays a `413 Request Entity Too Large` +error, the [max attachment size](#max-attachment-size) +is probably larger than the web server's allowed value. + +To increase the max attachment size to 200 MB in a +[Omnibus GitLab](https://docs.gitlab.com/omnibus/) install, you may need to +add the line below to `/etc/gitlab/gitlab.rb` before increasing the max attachment size: + +```ruby +nginx['client_max_body_size'] = "200m" +``` + +### This repository has exceeded its size limit + +If you receive intermittent push errors in your [Rails exceptions log](../../../administration/logs.md#exceptions_jsonlog), like this: + +```plaintext +Your push has been rejected, because this repository has exceeded its size limit. +``` + +[Housekeeping](../../../administration/housekeeping.md) tasks may be causing your repository size to grow. +To resolve this problem, either of these options helps in the short- to middle-term: + +- Increase the [repository size limit](#repository-size-limit). +- [Reduce the repo size](../../project/repository/reducing_the_repo_size_using_git.md). diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index 683c07460ee..170d3cf4c90 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -149,7 +149,7 @@ As an administrator you can set either a global or namespace-specific limit on t ## Archive jobs -Archiving jobs is useful for reducing the CI/CD footprint on the system by removing some +Archiving jobs is useful for reducing the CI/CD footprint on the system by removing some of the capabilities of the jobs (metadata stored in the database needed to run the job), but persisting the traces and artifacts for auditing purposes. @@ -170,7 +170,7 @@ For the value set for GitLab.com, see [Scheduled job archiving](../../gitlab_com ## Protect CI/CD variables by default To set all new [CI/CD variables](../../../ci/variables/index.md) as -[protected](../../../ci/variables/index.md#protect-a-cicd-variable) by default: +[protected](../../../ci/variables/index.md#protected-cicd-variables) by default: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > CI/CD**. @@ -224,13 +224,9 @@ To enable or disable the banner: 1. Select or clear the **Enable pipeline suggestion banner** checkbox. 1. Select **Save changes**. -## Required pipeline configuration **(PREMIUM SELF)** +## Required pipeline configuration **(ULTIMATE SELF)** -WARNING: -Required pipeline configurations is in its end-of-life process for Premium users. It's -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352316) in GitLab 14.8, -and planned to be unavailable for Premium users in GitLab 15.0. This feature is planned to continue -to be available for Ultimate users. Ultimate users are not impacted by this deprecation and removal. +> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/352316) from GitLab Premium to GitLab Ultimate in 15.0. NOTE: An alternative [compliance solution](../../project/settings/index.md#compliance-pipeline-configuration) diff --git a/doc/user/admin_area/settings/files_api_rate_limits.md b/doc/user/admin_area/settings/files_api_rate_limits.md index 7305e49b0d2..544c81e0583 100644 --- a/doc/user/admin_area/settings/files_api_rate_limits.md +++ b/doc/user/admin_area/settings/files_api_rate_limits.md @@ -8,7 +8,7 @@ type: reference # Rate limits on Repository files API **(FREE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68561) in GitLab 14.3. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75918) in GitLab 14.6. [Feature flag files_api_throttling](https://gitlab.com/gitlab-org/gitlab/-/issues/338903) removed. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75918) in GitLab 14.6. [Feature flag `files_api_throttling`](https://gitlab.com/gitlab-org/gitlab/-/issues/338903) removed. The [Repository files API](../../../api/repository_files.md) enables you to fetch, create, update, and delete files in your repository. To improve the security diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index 052b6e26c07..970897fd8da 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -112,8 +112,6 @@ The **Metrics and profiling** settings contain: - [Self monitoring](../../../administration/monitoring/gitlab_self_monitoring_project/index.md#create-the-self-monitoring-project) - Enable or disable instance self monitoring. - [Usage statistics](usage_statistics.md) - Enable or disable version check and Service Ping. -- [Pseudonymizer data collection](../../../administration/pseudonymizer.md) - - Enable or disable the Pseudonymizer data collection. ### Network @@ -179,8 +177,10 @@ The **Repository** settings contain: - Repository maintenance: - [Repository checks](../../../administration/repository_checks.md) - Configure automatic Git checks on repositories. - - [Housekeeping](../../../administration/housekeeping.md)). Configure automatic + - [Housekeeping](../../../administration/housekeeping.md). Configure automatic Git housekeeping on repositories. + - [Inactive project deletion](../../../administration/inactive_project_deletion.md). Configure inactive + project deletion. - [Repository static objects](../../../administration/static_objects_external_storage.md) - Serve repository static objects (for example, archives and blobs) from an external storage (for example, a CDN). diff --git a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md index 50dd24de3fb..6c0c15243da 100644 --- a/doc/user/admin_area/settings/rate_limit_on_issues_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_issues_creation.md @@ -15,7 +15,7 @@ To can change its value: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Network**. 1. Expand **Issues Rate Limits**. -1. Under **Max requests per minute per user**, enter the new value. +1. Under **Max requests per minute**, enter the new value. 1. Select **Save changes**. For example, if you set a limit of 300, requests using the diff --git a/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md new file mode 100644 index 00000000000..2819a18d361 --- /dev/null +++ b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md @@ -0,0 +1,33 @@ +--- +type: reference +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Rate limits on pipeline creation **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362475) in GitLab 15.0. + +You can set a limit so that users and processes can't request more than a certain number of pipelines each minute. This limit can help save resources and improve stability. + +For example, if you set a limit of `10`, and `11` requests are sent to the [trigger API](../../../ci/triggers/) within one minute, +the eleventh request is blocked. Access to the endpoint is allowed again after one minute. + +This limit is: + +- Applied independently per project, user, and commit. +- Not applied per IP address. +- Disabled by default. + +Requests that exceed the limit are logged in the `application_json.log` file. + +## Set a pipeline request limit + +To limit the number of pipeline requests: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Network**. +1. Expand **Pipelines Rate Limits**. +1. Under **Max requests per minute**, enter a value greater than `0`. +1. Select **Save changes**. diff --git a/doc/user/admin_area/settings/sign_in_restrictions.md b/doc/user/admin_area/settings/sign_in_restrictions.md index c63cd88eeb4..7316b1bdbb8 100644 --- a/doc/user/admin_area/settings/sign_in_restrictions.md +++ b/doc/user/admin_area/settings/sign_in_restrictions.md @@ -66,7 +66,7 @@ Git clients, and access RESTful API endpoints as administrators, without additio authentication steps. We may address these limitations in the future. For more information see the following epic: -[Admin mode for GitLab Administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158). +[Admin Mode for GitLab Administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158). ### Troubleshooting Admin Mode diff --git a/doc/user/admin_area/settings/sign_up_restrictions.md b/doc/user/admin_area/settings/sign_up_restrictions.md index 8ce3b4f1c18..534450c1871 100644 --- a/doc/user/admin_area/settings/sign_up_restrictions.md +++ b/doc/user/admin_area/settings/sign_up_restrictions.md @@ -60,7 +60,7 @@ To enforce confirmation of the email address used for new sign ups: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**, and expand **Sign-up restrictions**. -1. Select the **Enable email restrictions for sign ups** checkbox, then select **Save changes**. +1. Select the **Send confirmation email on sign-up** checkbox, then select **Save changes**. ## User cap diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 923ea9e19c1..ce949999fb8 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -1,8 +1,7 @@ --- -stage: none -group: unassigned +stage: Growth +group: Product Intelligence info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference --- # Usage statistics **(FREE SELF)** @@ -15,13 +14,54 @@ All usage statistics are [opt-out](#enable-or-disable-usage-statistics). ## Service Ping Service Ping is a process that collects and sends a weekly payload to GitLab Inc. -For more information, see the [Service Ping guide](../../../development/service_ping/index.md). +For more information, see the [Service Ping guide](../../../development/service_ping/index.md). When Service Ping is enabled, GitLab gathers data from other instances and enables certain [instance-level analytics features](../analytics/index.md) +that are dependent on Service Ping. -### Instance-level analytics availability +### Why enable Service Ping? -When Service Ping is enabled, GitLab gathers data from other instances and -enables certain [instance-level analytics features](../analytics/index.md) -that are dependent on Service Ping. +The main purpose of Service Ping is to build a better GitLab. We collect data about how GitLab is used +to understand feature or stage adoption and usage. This data gives an insight into how GitLab adds +value and helps our team understand the reasons why people use GitLab, and with this knowledge we're able to make better product decisions. + +There are several other benefits to enabling Service Ping: + +- Analyze the users' activities over time of your GitLab installation. +- A [DevOps Score](../analytics/dev_ops_reports.md#devops-score) to give you an overview of your entire instance's adoption of concurrent DevOps from planning to monitoring. +- More proactive support (assuming that our TAMs and support organization used the data to deliver more value). +- Insight and advice into how to get the most value out of your investment in GitLab. +- Reports that show how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. +- Participation in our [Registration Features Program](#registration-features-program) to receive free paid features. + +## Registration Features Program + +> Introduced in GitLab 14.1. + +In GitLab versions 14.1 and later, GitLab Free customers with a self-managed instance running +GitLab Enterprise Edition can receive paid features by registering with GitLab and sending us +activity data through Service Ping. Features introduced here do not remove the feature from its paid +tier. Users can continue to access the features in a paid tier without sharing usage data. + +### Features available in 14.1 and later + +- [Email from GitLab](../email_from_gitlab.md). + +### Features available in 14.4 and later + +- [Repository size limit](../settings/account_and_limit_settings.md#repository-size-limit). +- [Restrict group access by IP address](../../group/index.md#restrict-group-access-by-ip-address). + +NOTE: +Registration is not yet required for participation, but may be added in a future milestone. + +### Enable registration features + +1. Sign in as a user with administrator access. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. If not enabled, select the **Enable Service Ping** checkbox. +1. Select the **Enable Registration Features** checkbox. +1. Select **Save changes**. ## Version check @@ -79,6 +119,81 @@ To enable or disable Service Ping and version check: 1. Select or clear the **Enable version check** and **Enable Service Ping** checkboxes. 1. Select **Save changes**. +## Disable usage statistics with the configuration file + +NOTE: +The method to disable Service Ping in the GitLab configuration file does not work in +GitLab versions 9.3 to 13.12.3. For more information about how to disable it, see [troubleshooting](../../../development/service_ping/troubleshooting.md#cannot-disable-service-ping-with-the-configuration-file). + +To disable Service Ping and prevent it from being configured in the future through +the Admin Area: + +**For installations using the Linux package:** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['usage_ping_enabled'] = false + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +**For installations from source:** + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + # ... + gitlab: + # ... + usage_ping_enabled: false + ``` + +1. Restart GitLab: + + ```shell + sudo service gitlab restart + ``` + +## View the Service Ping payload + +You can view the exact JSON payload sent to GitLab Inc. in the Admin Area. To view the payload: + +1. Sign in as a user with administrator access. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. Select **Preview payload**. + +For an example payload, see [Example Service Ping payload](../../../development/service_ping/index.md#example-service-ping-payload). + +## Manually upload Service Ping payload + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7388) in GitLab 14.8 with a flag named `admin_application_settings_service_usage_data_center`. Disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83265) in GitLab 14.10. + +You can upload the Service Ping payload to GitLab even if your instance doesn't have internet access, +or if the Service Ping [cron job](../../../development/service_ping/index.md#how-service-ping-works) is not enabled. + +To upload the payload manually: + +1. Sign in as a user with administrator access. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Service** usage data. +1. Select **Download payload**. +1. Save the JSON file. +1. Visit [Service usage data center](https://version.gitlab.com/usage_data/new). +1. Select **Choose file** and choose the file from p5. +1. Select **Upload**. + +The uploaded file is encrypted and sent using secure HTTPS protocol. HTTPS creates a secure +communication channel between web browser and the server, and protects transmitted data against man-in-the-middle attacks. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/admin_area/settings/user_and_ip_rate_limits.md b/doc/user/admin_area/settings/user_and_ip_rate_limits.md index 56e240a8d39..bb3ee64abac 100644 --- a/doc/user/admin_area/settings/user_and_ip_rate_limits.md +++ b/doc/user/admin_area/settings/user_and_ip_rate_limits.md @@ -104,7 +104,7 @@ attached into the response headers. | Header | Example | Description | |:----------------------|:--------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `RateLimit-Limit` | `60` | The request quota for the client **each minute**. If the rate limit period set in the admin area is different from 1 minute, the value of this header is adjusted to approximately the nearest 60-minute period. | +| `RateLimit-Limit` | `60` | The request quota for the client **each minute**. If the rate limit period set in the Admin Area is different from 1 minute, the value of this header is adjusted to approximately the nearest 60-minute period. | | `RateLimit-Name` | `throttle_authenticated_web` | Name of the throttle blocking the requests. | | `RateLimit-Observed` | `67` | Number of requests associated to the client in the time window. | | `RateLimit-Remaining` | `0` | Remaining quota in the time window. The result of `RateLimit-Limit` - `RateLimit-Observed`. | diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 704476cdc90..017a0c46570 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -127,12 +127,6 @@ To retrieve metrics for change failure rate, use the [GraphQL](../../api/graphql We use the following terms to describe GitLab analytics: -- **Cycle time:** The duration of only the execution work. Cycle time is often displayed in combination with the lead time, which is longer than the cycle time. GitLab measures cycle time from the earliest commit of a [linked issue's merge request](../project/issues/crosslinking_issues.md) to when that issue is closed. The cycle time approach underestimates the lead time because merge request creation is always later than commit time. GitLab displays cycle time in [group-level Value Stream Analytics](../group/value_stream_analytics/index.md) and [project-level Value Stream Analytics](../analytics/value_stream_analytics.md). -- **Deploys:** The total number of successful deployments to production in the given time frame (across all applicable projects). GitLab displays deploys in [group-level Value Stream Analytics](../group/value_stream_analytics/index.md) and [project-level Value Stream Analytics](value_stream_analytics.md). -- **Lead time:** The duration of your value stream, from start to finish. Different to -[Lead time for changes](#lead-time-for-changes). Often displayed in combination with "cycle time," -which is shorter. GitLab measures lead time from issue creation to issue close. GitLab displays lead -time in [group-level Value Stream Analytics](../group/value_stream_analytics/index.md). - **Mean Time to Change (MTTC):** The average duration between idea and delivery. GitLab measures MTTC from issue creation to the issue's latest related merge request's deployment to production. - **Mean Time to Detect (MTTD):** The average duration that a bug goes undetected in production. @@ -142,11 +136,6 @@ merge request creation to merge request merge (and closed/un-merged merge reques For more information, see [Merge Request Analytics](merge_request_analytics.md). - **Mean Time to Recover/Repair/Resolution/Resolve/Restore (MTTR):** The average duration that a bug is not fixed in production. GitLab measures MTTR from deployment of bug to deployment of fix. -- **Throughput:** The number of issues closed or merge requests merged (not closed) in a period of -time. Often measured per sprint. GitLab displays merge request throughput in [Merge Request Analytics](merge_request_analytics.md). -- **Value Stream:** The entire work process that is followed to deliver value to customers. For example, -the [DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/) is a value stream that starts -with "plan" and ends with "monitor". GitLab helps you track your value stream using [Value Stream Analytics](value_stream_analytics.md). - **Velocity:** The total issue burden completed in some period of time. The burden is usually measured in points or weight, often per sprint. For example, your velocity may be "30 points per sprint". GitLab measures velocity as the total points or weight of issues closed in a given period of time. diff --git a/doc/user/analytics/merge_request_analytics.md b/doc/user/analytics/merge_request_analytics.md index 06774c3f16a..29f2ec79800 100644 --- a/doc/user/analytics/merge_request_analytics.md +++ b/doc/user/analytics/merge_request_analytics.md @@ -48,7 +48,8 @@ To view the number of merge requests merged per month: - In the **From** field, select a start date. - In the **To** field, select an end date. -The **Throughput** chart shows the number of merge requests merged per month. +The **Throughput** chart shows issues closed or merge requests merged (not closed) over a period of +time. The table shows up to 20 merge requests per page, and includes information about each merge request. @@ -58,9 +59,10 @@ information about each merge request. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229389) in GitLab 13.9. Use the number in **Mean time to merge** to view the average time between when a merge request is -created and when it's merged. +created and when it's merged. Closed and un-merged merge requests are not included. To view **Mean time to merge**: 1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Analytics > Merge request**. +1. On the left sidebar, select **Analytics > Merge request**. The **Mean time to merge** number +is shown on the dashboard. diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 92c4d447ed9..039d33a1ad8 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -12,6 +12,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w Value stream analytics provides metrics about each stage of your software development process. +A **value stream** is the entire work process that delivers value to customers. For example, +the [DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/) is a value stream that starts +with the Manage stage and ends with the Protect stage. + Use value stream analytics to identify: - The amount of time it takes to go from an idea to production. @@ -74,7 +78,7 @@ To view the median time spent in each stage: Value stream analytics shows the lead time and cycle time for issues in your project: - Lead time: Median time from when the issue was created to when it was closed. -- Cycle time: Median time from first commit to issue closed. Commits are associated with issues when users [cross-link them in the commit message](../project/issues/crosslinking_issues.md#from-commit-messages). +- Cycle time: Median time from first commit to issue closed. GitLab measures cycle time from the earliest commit of a [linked issue's merge request](../project/issues/crosslinking_issues.md) to when that issue is closed. The cycle time approach underestimates the lead time because merge request creation is always later than commit time. To view the lead time and cycle time for issues: diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index ed94686b7a3..f97e09f33bb 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -139,7 +139,7 @@ OpenAPI 2.x lets you specify the accepted media types globally or per operation, - In [GitLab 14.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/333304), the default behavior is to select one of the supported media types to use. The first supported media type is chosen from the list. This behavior is configurable. - In GitLab 14.9 and earlier, the default behavior is to perform testing using all supported media types. This means if two media types are listed (for example, `application/json` and `application/xml`), tests are performed using JSON, and then the same tests using XML. -Testing the same operation (for example, `POST /user`) using different media types (for example, `application/json` and `application/xml`) is not always desirable. +Testing the same operation (for example, `POST /user`) using different media types (for example, `application/json` and `application/xml`) is not always desirable. For example, if the target application executes the same code regardless of the request content type, it will take longer to finish the test session, and it may report duplicate vulnerabilities related to the request body depending on the target app. The environment variable `FUZZAPI_OPENAPI_ALL_MEDIA_TYPES` lets you specify whether or not to use all supported media types instead of one when generating requests for a given operation. When the environmental variable `FUZZAPI_OPENAPI_ALL_MEDIA_TYPES` is set to any value, API Fuzzing will try to generate requests for all supported media types instead of one in a given operation. This will cause testing to take longer as testing is repeated for each provided media type. @@ -1087,7 +1087,7 @@ You can provide the following properties to exclude specific parameters during t - `body-json`: Use this property to exclude specific JSON nodes from a request that uses the media type `application/json`. The property's value is an array, each entry of the array is a [JSON Path](https://goessner.net/articles/JsonPath/) expression. - `body-xml`: Use this property to exclude specific XML nodes from a request that uses media type `application/xml`. The property's value is an array, each entry of the array is a [XPath v2](https://www.w3.org/TR/xpath20/) expression. -The following JSON document is an example of the expected structure to exclude parameters. +The following JSON document is an example of the expected structure to exclude parameters. ```json { @@ -1155,11 +1155,11 @@ To exclude the `password` field in a request that uses `application/x-www-form-u The exclude parameters uses `body-form` when the request uses a content type `application/x-www-form-urlencoded`. -##### Excluding a specific JSON nodes using JSON Path +##### Excluding a specific JSON nodes using JSON Path To exclude the `schema` property in the root object, set the `body-json` property's value to an array with the JSON Path expression `[ "$.schema" ]`. -The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. +The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. For instance, the JSON document looks like this: ```json @@ -1168,13 +1168,13 @@ For instance, the JSON document looks like this: } ``` -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path, characters like `$`, `*`, `.` among others have special meaning. +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path, characters like `$`, `*`, `.` among others have special meaning. -##### Excluding multiple JSON nodes using JSON Path +##### Excluding multiple JSON nodes using JSON Path To exclude the property `password` on each entry of an array of `users` at the root level, set the `body-json` property's value to an array with the JSON Path expression `[ "$.users[*].paswword" ]`. -The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. +The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. For instance, the JSON document looks like this: @@ -1184,7 +1184,7 @@ For instance, the JSON document looks like this: } ``` -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. ##### Excluding an XML attribute @@ -1196,17 +1196,17 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/@isEnabled" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions, characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions, characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. ##### Excluding an XML element's text -To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. +To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. In the XPath expression `/credentials/username/text()`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. Last part has a `/` that refers to the current element, and uses a XPath function called `text()` which identifies the text of the current element. @@ -1214,17 +1214,17 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/username/text()" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. ##### Excluding an XML element -To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. +To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. In the XPath expression `/credentials/username`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. @@ -1232,7 +1232,7 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/username" ] } @@ -1242,21 +1242,21 @@ The exclude parameters uses `body-xml` when the request uses a content type `app ##### Excluding an XML node with namespaces -To exclude a XML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. +To exclude a XML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. -In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. +In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. The namespace name should have been defined in the XML document which is part of the body request. You may check the namespace in the specification document HAR, OpenAPI, or Postman Collection file. ```json { - "body-xml": [ + "body-xml": [ "/credentials/s:login" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. #### Using a JSON string @@ -1294,7 +1294,7 @@ variables: FUZZAPI_EXCLUDE_PARAMETER_FILE: api-fuzzing-exclude-parameters.json ``` -The `api-fuzzing-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). +The `api-fuzzing-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). ### Exclude URLS @@ -1348,7 +1348,7 @@ variables: ##### Excluding URL using regular expressions -In order to exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. +In order to exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. ```yaml variables: @@ -1467,7 +1467,7 @@ reported. Faults detected by API Fuzzing occur in the live web application, and require manual investigation to determine if they are vulnerabilities. Fuzzing faults are included as vulnerabilities with a severity of Unknown. To facilitate investigation of the fuzzing faults, detailed information is -provided about the HTTP messages sent and received along with a description of the modification(s) +provided about the HTTP messages sent and received along with a description of the modifications made. Follow these steps to view details of a fuzzing fault: diff --git a/doc/user/application_security/cluster_image_scanning/index.md b/doc/user/application_security/cluster_image_scanning/index.md index 293645b8de6..aba28a5ca89 100644 --- a/doc/user/application_security/cluster_image_scanning/index.md +++ b/doc/user/application_security/cluster_image_scanning/index.md @@ -28,10 +28,17 @@ To integrate GitLab with security scanners other than those listed here, see You can use cluster image scanning through the following methods: -- [The cluster image scanning analyzer](#use-the-cluster-image-scanning-analyzer) +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> +- [The cluster image scanning analyzer](#use-the-cluster-image-scanning-analyzer-removed) ([Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in GitLab 15.0. Use [the GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) instead.) +<!--- end_remove --> - [The GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) -## Use the cluster image scanning analyzer +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> + +## Use the cluster image scanning analyzer (removed) + +This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in GitLab 15.0. +Use [the GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) instead. You can use the cluster image scanning analyzer to run cluster image scanning with [GitLab CI/CD](../../../ci/index.md). To enable the cluster image scanning analyzer, [include the CI job](#configuration) @@ -277,6 +284,7 @@ Here's an example cluster image scanning report: } ``` +<!--- end_remove --> ## Cluster image scanning with the GitLab agent You can use the [GitLab agent](../../clusters/agent/index.md) to @@ -284,14 +292,12 @@ scan images from within your Kubernetes cluster and record the vulnerabilities i ### Prerequisites -- [Starboard Operator](https://aquasecurity.github.io/starboard/v0.10.3/operator/installation/kubectl/) - installed and configured in your cluster. - [GitLab agent](../../clusters/agent/install/index.md) set up in GitLab, installed in your cluster, and configured using a configuration repository. ### Configuration -The agent runs the cluster image scanning once the `cluster_image_scanning` +The agent runs the cluster image scanning once the `starboard` directive is added to your [agent's configuration repository](../../clusters/agent/vulnerabilities.md). ## Security Dashboard @@ -304,9 +310,12 @@ the security vulnerabilities in your groups, projects, and pipelines. After you find a vulnerability, you can address it in the [vulnerability report](../vulnerabilities/index.md) or the [GitLab agent's](../../clusters/agent/vulnerabilities.md) details section. +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> ## Troubleshooting ### Getting warning message `gl-cluster-image-scanning-report.json: no matching files` For information on this error, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). + +<!--- end_remove --> diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 64566e458ee..2828b56a5d1 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -5,7 +5,7 @@ group: Container Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Container Scanning **(ULTIMATE)** +# Container Scanning **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3672) in GitLab 10.4. @@ -44,6 +44,26 @@ information directly in the merge request.  +### Capabilities + +| Capability | In Free | In Ultimate | +| --- | ------ | ------ | +| [Configure Scanners](#configuration) | Yes | Yes | +| Customize Settings ([Variables](#available-cicd-variables), [Overriding](#overriding-the-container-scanning-template), [offline environment support](#running-container-scanning-in-an-offline-environment), etc) | Yes | Yes | +| [View JSON Report](#reports-json-format) as a CI job artifact | Yes | Yes | +| Generation of a JSON report of [dependencies](#dependency-list) as a CI job artifact | Yes | Yes | +| Ability to enable container scanning via an MR in the GitLab UI | Yes | Yes | +| [UBI Image Support](#fips-enabled-images) | Yes | Yes | +| Support for Trivy | Yes | Yes | +| Support for Grype | Yes | Yes | +| Inclusion of GitLab Advisory Database | Limited to the time-delayed content from GitLab [advisories-communities](https://gitlab.com/gitlab-org/advisories-community/) project | Yes - all the latest content from [Gemnasium DB](https://gitlab.com/gitlab-org/security-products/gemnasium-db) | +| Presentation of Report data in Merge Request and Security tab of the CI pipeline job | No | Yes | +| [Interaction with Vulnerabilities](#interacting-with-the-vulnerabilities) such as merge request approvals | No | Yes | +| [Solutions for vulnerabilities (auto-remediation)](#solutions-for-vulnerabilities-auto-remediation) | No | Yes | +| Support for the [vulnerability allow list](#vulnerability-allowlisting) | No | Yes | +| [Access to Security Dashboard page](#security-dashboard) | No | Yes | +| [Access to Dependency List page](../dependency_list/) | No | Yes | + ## Requirements To enable container scanning in your pipeline, you need the following: @@ -164,8 +184,8 @@ include: The `CS_DISABLE_DEPENDENCY_LIST` CI/CD variable controls whether the scan creates a [Dependency List](../dependency_list/) -report. The variable's default setting of `false` causes the scan to create the report. To disable -the report, set the variable to `true`: +report. This variable is currently only supported when the `trivy` analyzer is used. The variable's default setting of `"false"` causes the scan to create the report. To disable +the report, set the variable to `"true"`: For example: @@ -205,8 +225,9 @@ container_scanning: When you enable this feature, you may see [duplicate findings](../terminology/#duplicate-finding) in the [Vulnerability Report](../vulnerability_report/) if [Dependency Scanning](../dependency_scanning/) -is enabled for your project. This happens because GitLab can't automatically deduplicate the -findings reported by the two different analyzers. +is enabled for your project. This happens because GitLab can't automatically deduplicate findings +across different types of scanning tools. Please reference [this comparison](../dependency_scanning/#dependency-scanning-compared-to-container-scanning) +between GitLab Dependency Scanning and Container Scanning for more details on which types of dependencies are likely to be duplicated. #### Available CI/CD variables @@ -217,7 +238,7 @@ You can [configure](#customizing-the-container-scanning-settings) analyzers by u | `ADDITIONAL_CA_CERT_BUNDLE` | `""` | Bundle of CA certs that you want to trust. See [Using a custom SSL CA certificate authority](#using-a-custom-ssl-ca-certificate-authority) for more details. | All | | `CI_APPLICATION_REPOSITORY` | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | Docker repository URL for the image to be scanned. | All | | `CI_APPLICATION_TAG` | `$CI_COMMIT_SHA` | Docker repository tag for the image to be scanned. | All | -| `CS_ANALYZER_IMAGE` | `registry.gitlab.com/security-products/container-scanning:4` | Docker image of the analyzer. | All | +| `CS_ANALYZER_IMAGE` | `registry.gitlab.com/security-products/container-scanning:5` | Docker image of the analyzer. | All | | `CS_DEFAULT_BRANCH_IMAGE` | `""` | The name of the `DOCKER_IMAGE` on the default branch. See [Setting the default branch image](#setting-the-default-branch-image) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338877) in GitLab 14.5. | All | | `CS_DISABLE_DEPENDENCY_LIST` | `"false"` | Disable Dependency Scanning for packages installed in the scanned image. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345434) in GitLab 14.6. | All | | `CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN` | `"true"` | Disable scanning for language-specific packages installed in the scanned image. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345434) in GitLab 14.6. | All | @@ -234,10 +255,24 @@ You can [configure](#customizing-the-container-scanning-settings) analyzers by u ### Supported distributions -Support depends on the scanner: - -- [Grype](https://github.com/anchore/grype#grype) -- [Trivy](https://aquasecurity.github.io/trivy/latest/docs/vulnerability/detection/os/) (Default). +Support depends on which scanner is used: + +| Distribution | Grype | Trivy | +| -------------- | ----- | ----- | +| Alma Linux | | ✅ | +| Alpine Linux | ✅ | | +| Amazon Linux | ✅ | ✅ | +| BusyBox | ✅ | | +| CentOS | ✅ | ✅ | +| CBL-Mariner | | ✅ | +| Debian | ✅ | ✅ | +| Distroless | ✅ | ✅ | +| Oracle Linux | ✅ | ✅ | +| Photon OS | | ✅ | +| Red Hat (RHEL) | ✅ | ✅ | +| Rocky Linux | | ✅ | +| SUSE | | ✅ | +| Ubuntu | ✅ | ✅ | #### FIPS-enabled images @@ -250,9 +285,9 @@ standard tag plus the `-fips` extension. | Scanner name | `CS_ANALYZER_IMAGE` | | --------------- | ------------------- | -| Default (Trivy) | `registry.gitlab.com/security-products/container-scanning:4-fips` | -| Grype | `registry.gitlab.com/security-products/container-scanning/grype:4-fips` | -| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:4-fips` | +| Default (Trivy) | `registry.gitlab.com/security-products/container-scanning:5-fips` | +| Grype | `registry.gitlab.com/security-products/container-scanning/grype:5-fips` | +| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:5-fips` | NOTE: Prior to GitLab 15.0, the `-ubi` image extension is also available. GitLab 15.0 and later only @@ -305,9 +340,9 @@ The following options are available: | Scanner name | `CS_ANALYZER_IMAGE` | | ------------ | ------------------- | -| Default ([Trivy](https://github.com/aquasecurity/trivy)) | `registry.gitlab.com/security-products/container-scanning:4` | -| [Grype](https://github.com/anchore/grype) | `registry.gitlab.com/security-products/container-scanning/grype:4` | -| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:4` | +| Default ([Trivy](https://github.com/aquasecurity/trivy)) | `registry.gitlab.com/security-products/container-scanning:5` | +| [Grype](https://github.com/anchore/grype) | `registry.gitlab.com/security-products/container-scanning/grype:5` | +| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:5` | If you're migrating from a GitLab 13.x release to a GitLab 14.x release and have customized the `container_scanning` job or its CI variables, you might need to perform these migration steps in @@ -320,7 +355,7 @@ your CI file: - `SECURE_ANALYZERS_PREFIX` 1. Review the `CS_ANALYZER_IMAGE` variable. It no longer depends on the variables above and its new - default value is `registry.gitlab.com/security-products/container-scanning:4`. If you have an + default value is `registry.gitlab.com/security-products/container-scanning:5`. If you have an offline environment, see [Running container scanning in an offline environment](#running-container-scanning-in-an-offline-environment). @@ -405,7 +440,7 @@ container_scanning: The `ADDITIONAL_CA_CERT_BUNDLE` value can also be configured as a [custom variable in the UI](../../../ci/variables/index.md#custom-cicd-variables), either as a `file`, which requires the path to the certificate, or as a variable, which requires the text representation of the certificate. -### Vulnerability allowlisting +### Vulnerability allowlisting **(ULTIMATE)** To allowlist specific vulnerabilities, follow these steps: @@ -532,9 +567,9 @@ For container scanning, import the following images from `registry.gitlab.com` i [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/security-products/container-scanning:4 -registry.gitlab.com/security-products/container-scanning/grype:4 -registry.gitlab.com/security-products/container-scanning/trivy:4 +registry.gitlab.com/security-products/container-scanning:5 +registry.gitlab.com/security-products/container-scanning/grype:5 +registry.gitlab.com/security-products/container-scanning/trivy:5 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -574,7 +609,7 @@ following `.gitlab-ci.yml` example as a template. ```yaml variables: - SOURCE_IMAGE: registry.gitlab.com/security-products/container-scanning:4 + SOURCE_IMAGE: registry.gitlab.com/security-products/container-scanning:5 TARGET_IMAGE: $CI_REGISTRY/namespace/gitlab-container-scanning image: docker:stable @@ -753,15 +788,38 @@ Here's an example container scanning report: The [Security Dashboard](../security_dashboard/index.md) shows you an overview of all the security vulnerabilities in your groups, projects and pipelines. -## Vulnerabilities database update +## Vulnerabilities database All analyzer images are [updated daily](https://gitlab.com/gitlab-org/security-products/analyzers/container-scanning/-/blob/master/README.md#image-updates). -The images include the latest advisory database available for their respective scanner. Each -scanner includes data from multiple sources: - -- [Grype](https://github.com/anchore/grype#grypes-database). -- [Trivy](https://aquasecurity.github.io/trivy/latest/docs/vulnerability/detection/data-source/). +The images use data from upstream advisory databases depending on which scanner is used: + +| Data Source | Trivy | Grype | +| ------------------------------ | ----- | ----- | +| AlmaLinux Security Advisory | ✅ | ✅ | +| Amazon Linux Security Center | ✅ | ✅ | +| Arch Linux Security Tracker | ✅ | | +| SUSE CVRF | ✅ | ✅ | +| CWE Advisories | ✅ | | +| Debian Security Bug Tracker | ✅ | ✅ | +| GitHub Security Advisory | ✅ | ✅ | +| Go Vulnerability Database | ✅ | | +| CBL-Mariner Vulnerability Data | ✅ | | +| NVD | ✅ | ✅ | +| OSV | ✅ | | +| Red Hat OVAL v2 | ✅ | ✅ | +| Red Hat Security Data API | ✅ | ✅ | +| Photon Security Advisories | ✅ | | +| Rocky Linux UpdateInfo | ✅ | | +| Ubuntu CVE Tracker (only data sources from mid 2021 and later) | ✅ | ✅ | + +In addition to the sources provided by these scanners, GitLab maintains the following vulnerability databases: + +- The proprietary +[GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db). +- The open source [GitLab Advisory Database (Open Source Edition)](https://gitlab.com/gitlab-org/advisories-community). + +In the GitLab Ultimate tier, the data from the [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is merged in to augment the data from the external sources. In the GitLab Premium and Free tiers, the data from the [GitLab Advisory Database (Open Source Edition)](https://gitlab.com/gitlab-org/advisories-community) is merged in to augment the data from the external sources. This augmentation currently only applies to the analyzer images for the Trivy scanner. Database update information for other analyzers is available in the [maintenance table](../index.md#vulnerability-scanner-maintenance). @@ -770,7 +828,7 @@ Database update information for other analyzers is available in the After a vulnerability is found, you can [address it](../vulnerabilities/index.md). -## Solutions for vulnerabilities (auto-remediation) +## Solutions for vulnerabilities (auto-remediation) **(ULTIMATE)** Some vulnerabilities can be fixed by applying the solution that GitLab automatically generates. @@ -827,6 +885,7 @@ For information on this, see the [general Application Security troubleshooting s as the default for container scanning, and also [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326279) an integration with [Grype](https://github.com/anchore/grype) as an alternative scanner. +- GitLab 15.0 changed the major analyzer version from `4` to `5`. Other changes to the container scanning analyzer can be found in the project's [changelog](https://gitlab.com/gitlab-org/security-products/analyzers/container-scanning/-/blob/master/CHANGELOG.md). diff --git a/doc/user/application_security/dast/checks/16.3.md b/doc/user/application_security/dast/checks/16.3.md index e4fc2468dae..6f80a2a32c6 100644 --- a/doc/user/application_security/dast/checks/16.3.md +++ b/doc/user/application_security/dast/checks/16.3.md @@ -32,4 +32,4 @@ information from the `X-Powered-By` header. ## Links - [CWE](https://cwe.mitre.org/data/definitions/16.html) -- [PHP expose_php](https://www.php.net/manual/en/ini.core.php#ini.expose-php) +- [PHP `expose_php`](https://www.php.net/manual/en/ini.core.php#ini.expose-php) diff --git a/doc/user/application_security/dast/checks/16.5.md b/doc/user/application_security/dast/checks/16.5.md index 28bb9f7ee4b..e03da3043ef 100644 --- a/doc/user/application_security/dast/checks/16.5.md +++ b/doc/user/application_security/dast/checks/16.5.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Description -The target website returns AspNet header(s) and version information of this website. By +The target website returns AspNet headers and version information of this website. By exposing these values attackers may attempt to identify if the target software is vulnerable to known vulnerabilities, or catalog known sites running particular versions to exploit in the future when a vulnerability is identified in the particular version. diff --git a/doc/user/application_security/dast/checks/16.6.md b/doc/user/application_security/dast/checks/16.6.md index ddd3a10c5f8..9cbcde669a0 100644 --- a/doc/user/application_security/dast/checks/16.6.md +++ b/doc/user/application_security/dast/checks/16.6.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Description -The target website returns AspNet header(s) along with version information of this website. By +The target website returns AspNet headers along with version information of this website. By exposing these values attackers may attempt to identify if the target software is vulnerable to known vulnerabilities. Or catalog known sites running particular versions to exploit in the future when a vulnerability is identified in the particular version. diff --git a/doc/user/application_security/dast/checks/359.1.md b/doc/user/application_security/dast/checks/359.1.md new file mode 100644 index 00000000000..af1fdf8a596 --- /dev/null +++ b/doc/user/application_security/dast/checks/359.1.md @@ -0,0 +1,34 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Exposure of Private Personal Information (PII) to an unauthorized actor (credit card) + +## Description + +The target application was found to return credit card information in the response. Organizations +found returning such information may be in violation of industry regulations and could face fines. + +## Remediation + +PII such as credit cards should never be directly returned to the user. The majority of the information should masked except +the last few digits or characters of the identifier. For example, credit card numbers should +only return the last four digits: `****-****-****-1234`. Ensure this masking is done on the server +and only then send the masked data back to the client. Do not rely on client side JavaScript or other methods +to mask these values as the data could still be intercepted or unmasked. + +Additionally, credit card information should never be stored un-encrypted in files or databases. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 359.1 | true | 359 | Passive | Medium | + +## Links + +- [OWASP Top 10 A3 2017 - Sensitive Data Exposure](https://owasp.org/www-project-top-ten/2017/A3_2017-Sensitive_Data_Exposure) +- [CWE](https://cwe.mitre.org/data/definitions/359.html) +- [PCI-DSS](https://www.pcisecuritystandards.org/pdfs/pci_fs_data_storage.pdf) diff --git a/doc/user/application_security/dast/checks/359.2.md b/doc/user/application_security/dast/checks/359.2.md new file mode 100644 index 00000000000..beb99e26097 --- /dev/null +++ b/doc/user/application_security/dast/checks/359.2.md @@ -0,0 +1,34 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Exposure of Private Personal Information (PII) to an unauthorized actor (United States social security number) + +## Description + +The target application was found to return social security number (SSN) information in the response. Organizations +found returning such information may be in violation of (United States) state or federal laws and may face stiff penalties. + +## Remediation + +PII such as social security numbers should never be directly returned to the user. The majority of the information +should masked except the last few digits or characters of the identifier. For example, social security numbers +only be displayed with the last four digits: `***-**-1234`. Ensure this masking is done on the server +and only then send the masked data back to the client. Do not rely on client side JavaScript or other methods +to mask these values as the data could still be intercepted or unmasked. + +Additionally, social security numbers should never be stored un-encrypted in files or databases. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 359.2 | true | 359 | Passive | Medium | + +## Links + +- [OWASP Top 10 A3 2017 - Sensitive Data Exposure](https://owasp.org/www-project-top-ten/2017/A3_2017-Sensitive_Data_Exposure) +- [CWE](https://cwe.mitre.org/data/definitions/359.html) +- [Privacy Act (CMPPA)](https://www.ssa.gov/dataexchange/privacyinfo.html) diff --git a/doc/user/application_security/dast/checks/index.md b/doc/user/application_security/dast/checks/index.md index 764e3c4a839..629ff1c3a8d 100644 --- a/doc/user/application_security/dast/checks/index.md +++ b/doc/user/application_security/dast/checks/index.md @@ -18,6 +18,8 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [16.5](16.5.md) | AspNet header exposes version information | Low | Passive | | [16.6](16.6.md) | AspNetMvc header exposes version information | Low | Passive | | [200.1](200.1.md) | Exposure of sensitive information to an unauthorized actor (private IP address) | Low | Passive | +| [359.1](359.1.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (credit card) | Medium | Passive | +| [359.2](359.2.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (United States social security number) | Medium | Passive | | [548.1](548.1.md) | Exposure of information through directory listing | Low | Passive | | [598.1](598.1.md) | Use of GET request method with sensitive query strings (session ID) | Medium | Passive | | [598.2](598.2.md) | Use of GET request method with sensitive query strings (password) | Medium | Passive | diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index ee57803dfc7..1389db65713 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -627,7 +627,7 @@ These CI/CD variables are specific to DAST. They can be used to customize the be | `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | | `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | | `DAST_API_OPENAPI` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/290241) in GitLab 13.12 and replaced by `DAST_API_OPENAPI`. To be removed in GitLab 15.0. The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/290241)** in GitLab 15.0. Replaced by `DAST_API_OPENAPI`. The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | | `DAST_AUTH_REPORT` <sup>2</sup> | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | | `DAST_AUTH_EXCLUDE_URLS` <sup>2</sup> | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. | | `DAST_AUTH_URL` <sup>1,2</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. Example: `https://login.example.com`. | @@ -1156,6 +1156,7 @@ A site profile contains the following: - **Password**: The password used to authenticate to the website. - **Username form field**: The name of username field at the sign-in HTML form. - **Password form field**: The name of password field at the sign-in HTML form. + - **Submit form field**: The `id` or `name` of the element that when clicked submits the sign-in HTML form. When an API site type is selected, a [host override](#host-override) is used to ensure the API being scanned is on the same host as the target. This is done to reduce the risk of running an active scan against the wrong API. @@ -1199,7 +1200,14 @@ The site profile is created. #### Edit a site profile -To edit an existing site profile: +If a site profile is linked to a security policy, a user cannot edit the profile from this page. See +[Scan execution policies](../policies/scan-execution-policies.md) +for more information. + +When a validated site profile's file, header, or meta tag is edited, the site's +[validation status](#site-profile-validation) is revoked. + +To edit a site profile: 1. From your project's home page, go to **Security & Compliance > Configuration**. 1. In the **DAST Profiles** row select **Manage**. @@ -1207,42 +1215,37 @@ To edit an existing site profile: 1. In the profile's row select the **More actions** (**{ellipsis_v}**) menu, then select **Edit**. 1. Edit the fields then select **Save profile**. -If a site profile is linked to a security policy, a user cannot edit the profile from this page. See -[Scan execution policies](../policies/scan-execution-policies.md) -for more information. - #### Delete a site profile -To delete an existing site profile: +If a site profile is linked to a security policy, a user cannot delete the profile from this page. +See [Scan execution policies](../policies/scan-execution-policies.md) +for more information. + +To delete a site profile: 1. From your project's home page, go to **Security & Compliance > Configuration**. 1. In the **DAST Profiles** row select **Manage**. 1. Select the **Site Profiles** tab. -1. In the profile's row select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**. +1. In the profile's row, select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**. 1. Select **Delete** to confirm the deletion. -If a site profile is linked to a security policy, a user cannot delete the profile from this page. -See [Scan execution policies](../policies/scan-execution-policies.md) -for more information. - #### Validate a site profile -Prerequisites: - -- A site profile. +Validating a site is required to run an active scan. To validate a site profile: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Configuration**. -1. In the **Dynamic Application Security Testing (DAST)** section, select **Manage scans**. +1. In the **Dynamic Application Security Testing (DAST)** section, select **Manage profiles**. 1. Select the **Site Profiles** tab. -1. In the profile's row select **Validate** or **Retry validation**. +1. In the profile's row, select **Validate**. 1. Select the validation method. 1. For **Text file validation**: 1. Download the validation file listed in **Step 2**. - 1. Upload the validation file to the host. Upload the file to the location in - **Step 3** or any location you prefer. + 1. Upload the validation file to the host, to the location in **Step 3** or any location you + prefer. + 1. If required, edit the file location in **Step 3**. 1. Select **Validate**. 1. For **Header validation**: 1. Select the clipboard icon in **Step 2**. @@ -1255,9 +1258,8 @@ To validate a site profile: 1. Select the input field in **Step 3** and enter the location of the meta tag. 1. Select **Validate**. -The site is validated and an active scan can run against it. - -If a validated site profile's target URL is edited, the site's validation status is revoked. +The site is validated and an active scan can run against it. A site profile's validation status is +revoked only when it's revoked manually, or its file, header, or meta tag is edited. #### Retry a failed validation @@ -1265,22 +1267,28 @@ If a validated site profile's target URL is edited, the site's validation status > - [Deployed behind the `dast_failed_site_validations` flag](../../../administration/feature_flags.md), enabled by default. > - [Feature flag `dast_failed_site_validations` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323961) in GitLab 14.4. -If a site profile's validation fails, you can retry it by selecting the **Retry validation** button -in the profiles list. +Failed site validation attempts are listed on the **Site profiles** tab of the **Manage profiles** +page. + +To retry a site profile's failed validation: -When loading the DAST profiles library, past failed validations are listed above the profiles -list. You can also retry the validation from there by selecting the **Retry validation** link in -the alert. You can also dismiss the alert to revoke failed validations. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Configuration**. +1. In the **Dynamic Application Security Testing (DAST)** section, select **Manage profiles**. +1. Select the **Site Profiles** tab. +1. In the profile's row, select **Retry validation**. #### Revoke a site profile's validation status -Note that all site profiles with the same URL have their validation status revoked. +WARNING: +When a site profile's validation status is revoked, all site profiles that share the same URL also +have their validation status revoked. To revoke a site profile's validation status: 1. From your project's home page, go to **Security & Compliance > Configuration**. 1. In the **DAST Profiles** row select **Manage**. -1. Select **Revoke validation** beside the validated profile. +1. Beside the validated profile, select **Revoke validation**. The site profile's validation status is revoked. @@ -1348,40 +1356,40 @@ A scanner profile defines the scanner settings used to run an on-demand scan: To create a scanner profile: 1. From your project's home page, go to **Security & Compliance > Configuration**. -1. In the **DAST Profiles** row select **Manage**. +1. In the **DAST Profiles** row, select **Manage**. 1. Select **New > Scanner Profile**. 1. Complete the form. For details of each field, see [Scanner profile](#scanner-profile). -1. Click **Save profile**. +1. Select **Save profile**. #### Edit a scanner profile +If a scanner profile is linked to a security policy, a user cannot edit the profile from this page. +See [Scan execution policies](../policies/scan-execution-policies.md) +for more information. + To edit a scanner profile: 1. From your project's home page, go to **Security & Compliance > Configuration**. -1. Click **Manage** in the **DAST Profiles** row. +1. In the **DAST Profiles** row, select **Manage**. 1. Select the **Scanner Profiles** tab. -1. In the scanner's row select the **More actions** (**{ellipsis_v}**) menu, then select **Edit**. +1. In the scanner's row, select the **More actions** (**{ellipsis_v}**) menu, then select **Edit**. 1. Edit the form. 1. Select **Save profile**. -If a scanner profile is linked to a security policy, a user cannot edit the profile from this page. -See [Scan execution policies](../policies/scan-execution-policies.md) -for more information. - #### Delete a scanner profile +If a scanner profile is linked to a security policy, a user cannot delete the profile from this +page. See [Scan execution policies](../policies/scan-execution-policies.md) +for more information. + To delete a scanner profile: 1. From your project's home page, go to **Security & Compliance > Configuration**. -1. Click **Manage** in the **DAST Profiles** row. +1. In the **DAST Profiles** row, select **Manage**. 1. Select the **Scanner Profiles** tab. -1. In the scanner's row select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**. +1. In the scanner's row, select the **More actions** (**{ellipsis_v}**) menu, then select **Delete**. 1. Select **Delete**. -If a scanner profile is linked to a security policy, a user cannot delete the profile from this -page. See [Scan execution policies](../policies/scan-execution-policies.md) -for more information. - ## Auditing > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1. diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index a4908204b60..a1b19c52b20 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -1041,7 +1041,7 @@ You can provide the following properties to exclude specific parameters during t - `body-json`: Use this property to exclude specific JSON nodes from a request that uses the media type `application/json`. The property's value is an array, each entry of the array is a [JSON Path](https://goessner.net/articles/JsonPath/) expression. - `body-xml`: Use this property to exclude specific XML nodes from a request that uses media type `application/xml`. The property's value is an array, each entry of the array is a [XPath v2](https://www.w3.org/TR/xpath20/) expression. -Thus, the following JSON document is an example of the expected structure to exclude parameters. +Thus, the following JSON document is an example of the expected structure to exclude parameters. ```json { @@ -1109,11 +1109,11 @@ To exclude the `password` field in a request that uses `application/x-www-form-u The exclude parameters uses `body-form` when the request uses a content type `application/x-www-form-urlencoded`. -##### Excluding a specific JSON nodes using JSON Path +##### Excluding a specific JSON nodes using JSON Path To exclude the `schema` property in the root object, set the `body-json` property's value to an array with the JSON Path expression `[ "$.schema" ]`. -The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. +The JSON Path expression uses special syntax to identify JSON nodes: `$` refers to the root of the JSON document, `.` refers to the current object (in our case the root object), and the text `schema` refers to a property name. Thus, the JSON path expression `$.schema` refers to a property `schema` in the root object. For instance, the JSON document looks like this: ```json @@ -1122,13 +1122,13 @@ For instance, the JSON document looks like this: } ``` -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. -##### Excluding multiple JSON nodes using JSON Path +##### Excluding multiple JSON nodes using JSON Path To exclude the property `password` on each entry of an array of `users` at the root level, set the `body-json` property's value to an array with the JSON Path expression `[ "$.users[*].paswword" ]`. -The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. +The JSON Path expression starts with `$` to refer to the root node and uses `.` to refer to the current node. Then, it uses `users` to refer to a property and the characters `[` and `]` to enclose the index in the array you want to use, instead of providing a number as an index you use `*` to specify any index. After the index reference, we find `.` which now refers to any given selected index in the array, preceded by a property name `password`. For instance, the JSON document looks like this: @@ -1138,7 +1138,7 @@ For instance, the JSON document looks like this: } ``` -The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. +The exclude parameters uses `body-json` when the request uses a content type `application/json`. Each entry in `body-json` is expected to be a [JSON Path expression](https://goessner.net/articles/JsonPath/). In JSON Path characters like `$`, `*`, `.` among others have special meaning. ##### Excluding a XML attribute @@ -1150,17 +1150,17 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/@isEnabled" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. ##### Excluding a XML text's element -To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. +To exclude the text of the `username` element contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username/text()" ]`. In the XPath expression `/credentials/username/text()`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. Last part has a `/` that refers to the current element, and uses a XPath function called `text()` which identifies the text of the current element. @@ -1168,17 +1168,17 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/username/text()" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. ##### Excluding an XML element -To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. +To exclude the element `username` contained in root node `credentials`, set the `body-xml` property's value to an array with the XPath expression `[/credentials/username" ]`. In the XPath expression `/credentials/username`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `username`. @@ -1186,31 +1186,31 @@ For instance, the JSON document looks like this: ```json { - "body-xml": [ + "body-xml": [ "/credentials/username" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be a [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. ##### Excluding an XML node with namespaces -To exclude anXML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. +To exclude anXML element `login` which is defined in namespace `s`, and contained in `credentials` root node, set the `body-xml` property's value to an array with the XPath expression `[ "/credentials/s:login" ]`. -In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. +In the XPath expression `/credentials/s:login`, the first character `/` refers to the root XML node, and then after it indicates an XML element's name `credentials`. Similarly, the character `/` refers to the current element, followed by a new XML element's name `s:login`. Notice that name contains the character `:`, this character separates the namespace from the node name. The namespace name should have been defined in the XML document which is part of the body request. You may check the namespace in the specification document HAR, OpenAPI, or Postman Collection file. ```json { - "body-xml": [ + "body-xml": [ "/credentials/s:login" ] } ``` -The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath, expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. +The exclude parameters uses `body-xml` when the request uses a content type `application/xml`. Each entry in `body-xml` is expected to be an [XPath v2 expression](https://www.w3.org/TR/xpath20/). In XPath, expressions characters like `@`, `/`, `:`, `[`, `]` among others have special meanings. #### Using a JSON string @@ -1248,7 +1248,7 @@ variables: DAST_API_EXCLUDE_PARAMETER_FILE: dast-api-exclude-parameters.json ``` -The `dast-api-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). +The `dast-api-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). ### Exclude URLS @@ -1302,7 +1302,7 @@ variables: ##### Excluding URL using regular expressions -In order to exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. +In order to exclude exactly `https://target/api/v1/user/create` and `https://target/api/v2/user/create` or any other version (`v3`,`v4`, and more). We could use `https://target/api/v.*/user/create$`, in the previous regular expression `.` indicates any character and `*` indicates zero or more times, additionally `$` indicates that the URL should end there. ```yaml variables: diff --git a/doc/user/application_security/dependency_scanning/analyzers.md b/doc/user/application_security/dependency_scanning/analyzers.md index 665d29c4017..acbc94cba47 100644 --- a/doc/user/application_security/dependency_scanning/analyzers.md +++ b/doc/user/application_security/dependency_scanning/analyzers.md @@ -20,11 +20,9 @@ This is achieved by implementing the [common API](https://gitlab.com/gitlab-org/ Dependency Scanning supports the following official analyzers: -- [`bundler-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) - [`gemnasium`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) - [`gemnasium-maven`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven) - [`gemnasium-python`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python) -- [`retire.js`](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) The analyzers are published as Docker images, which Dependency Scanning uses to launch dedicated containers for each analysis. @@ -34,11 +32,13 @@ The Dependency Scanning analyzers' current major version number is 2. Dependency Scanning is pre-configured with a set of **default images** that are maintained by GitLab, but users can also integrate their own **custom images**. -WARNING: -The `bundler-audit` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#bundler-audit-dependency-scanning-tool). +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> -WARNING: -The `retire.js` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#retire-js-dependency-scanning-tool). +The [`bundler-audit`](https://gitlab.com/gitlab-org/gitlab/-/issues/289832) and [`retire.js`](https://gitlab.com/gitlab-org/gitlab/-/issues/350510) analyzers were deprecated +in GitLab 14.8 and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86704) in 15.0. +Use Gemnasium instead. + +<!--- end_remove --> ## Official default analyzers @@ -67,7 +67,7 @@ the official analyzers. ### Disable specific analyzers You can select the official analyzers you don't want to run. Here's how to disable -`bundler-audit` and `gemnasium` analyzers. +the `gemnasium` analyzer. In `.gitlab-ci.yml` define: ```yaml @@ -75,7 +75,7 @@ include: template: Security/Dependency-Scanning.gitlab-ci.yml variables: - DS_EXCLUDED_ANALYZERS: "bundler-audit, gemnasium" + DS_EXCLUDED_ANALYZERS: "gemnasium" ``` ### Disabling default analyzers @@ -88,7 +88,7 @@ include: template: Security/Dependency-Scanning.gitlab-ci.yml variables: - DS_EXCLUDED_ANALYZERS: "gemnasium, gemnasium-maven, gemnasium-python, bundler-audit, retire.js" + DS_EXCLUDED_ANALYZERS: "gemnasium, gemnasium-maven, gemnasium-python" ``` This is used when one totally relies on [custom analyzers](#custom-analyzers). @@ -117,25 +117,25 @@ The [Security Scanner Integration](../../../development/integrations/secure.md) ## Analyzers data -The following table lists the data available for each official analyzer. - -| Property \ Tool | Gemnasium | bundler-audit | Retire.js | -|---------------------------------------|:------------------:|:------------------:|:------------------:| -| Severity | 𐄂 | ✓ | ✓ | -| Title | ✓ | ✓ | ✓ | -| File | ✓ | ⚠ | ✓ | -| Start line | 𐄂 | 𐄂 | 𐄂 | -| End line | 𐄂 | 𐄂 | 𐄂 | -| External ID (for example, CVE) | ✓ | ✓ | ⚠ | -| URLs | ✓ | ✓ | ✓ | -| Internal doc/explanation | ✓ | 𐄂 | 𐄂 | -| Solution | ✓ | ✓ | 𐄂 | -| Confidence | 𐄂 | 𐄂 | 𐄂 | -| Affected item (for example, class or package) | ✓ | ✓ | ✓ | -| Source code extract | 𐄂 | 𐄂 | 𐄂 | -| Internal ID | ✓ | 𐄂 | 𐄂 | -| Date | ✓ | 𐄂 | 𐄂 | -| Credits | ✓ | 𐄂 | 𐄂 | +The following table lists the data available for the Gemnasium analyzer. + +| Property \ Tool | Gemnasium | +|---------------------------------------|:------------------:| +| Severity | 𐄂 | +| Title | ✓ | +| File | ✓ | +| Start line | 𐄂 | +| End line | 𐄂 | +| External ID (for example, CVE) | ✓ | +| URLs | ✓ | +| Internal doc/explanation | ✓ | +| Solution | ✓ | +| Confidence | 𐄂 | +| Affected item (for example, class or package) | ✓ | +| Source code extract | 𐄂 | +| Internal ID | ✓ | +| Date | ✓ | +| Credits | ✓ | - ✓ => we have that data - ⚠ => we have that data, but it's partially reliable, or we need to extract that data from unstructured content diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 924e3838d91..87d49ffa324 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -17,9 +17,11 @@ aspects of inspecting the items your code uses. These items typically include ap dependencies that are almost always imported from external sources, rather than sourced from items you wrote yourself. +## Dependency Scanning compared to Container Scanning + GitLab offers both Dependency Scanning and Container Scanning to ensure coverage for all of these dependency types. To cover as much of your risk area as -possible, we encourage you to use all of our security scanners: +possible, we encourage you to use all of our security scanning tools: - Dependency Scanning analyzes your project and tells you which software dependencies, including upstream dependencies, have been included in your project, and what known @@ -41,6 +43,21 @@ possible, we encourage you to use all of our security scanners: efforts to de-duplicate these findings can be tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/348655). +The following table summarizes which types of dependencies each scanning tool can detect: + +| Feature | Dependency Scanning | Container Scanning | +| ----------------------------------------------------------- | ------------------- | ------------------ | +| Identify the manifest, lock file, or static file that introduced the dependency | **{check-circle}** | **{dotted-circle}** | +| Development dependencies | **{check-circle}** | **{dotted-circle}** | +| Dependencies in a lock file committed to your repository | **{check-circle}** | **{check-circle}** <sup>1</sup> | +| Binaries built by Go | **{dotted-circle}** | **{check-circle}** <sup>2</sup> | +| Dynamically-linked language-specific dependencies installed by the Operating System | **{dotted-circle}** | **{check-circle}** | +| Operating system dependencies | **{dotted-circle}** | **{check-circle}** | +| Language-specific dependencies installed on the operating system (not built by your project) | **{dotted-circle}** | **{check-circle}** | + +1. Lock file must be present in the image to be detected. +1. Binary file must be present in the image to be detected. + ## Overview If you're using [GitLab CI/CD](../../../ci/index.md), you can use dependency scanning to analyze @@ -136,9 +153,9 @@ table.supported-languages ul { </thead> <tbody> <tr> - <td rowspan="2">Ruby</td> - <td rowspan="2">N/A</td> - <td rowspan="2"><a href="https://bundler.io/">Bundler</a></td> + <td>Ruby</td> + <td>N/A</td> + <td><a href="https://bundler.io/">Bundler</a></td> <td> <ul> <li><code>Gemfile.lock</code></li> @@ -149,11 +166,6 @@ table.supported-languages ul { <td>Y</td> </tr> <tr> - <td><code>Gemfile.lock</code></td> - <td><a href="https://github.com/rubysec/bundler-audit">bundler-audit</a></td> - <td>N</td> - </tr> - <tr> <td>PHP</td> <td>N/A</td> <td><a href="https://getcomposer.org/">Composer</a></td> @@ -200,9 +212,9 @@ table.supported-languages ul { <td>N</td> </tr> <tr> - <td rowspan="3">JavaScript</td> - <td rowspan="2">N/A</td> - <td rowspan="2"><a href="https://www.npmjs.com/">npm</a></td> + <td rowspan="2">JavaScript</td> + <td>N/A</td> + <td><a href="https://www.npmjs.com/">npm</a></td> <td> <ul> <li><code>package-lock.json</code></li> @@ -213,11 +225,6 @@ table.supported-languages ul { <td>Y</td> </tr> <tr> - <td><code>package.json</code></td> - <td><a href="https://retirejs.github.io/retire.js/">Retire.js</a></td> - <td>N</td> - </tr> - <tr> <td>N/A</td> <td><a href="https://classic.yarnpkg.com/en/">yarn</a></td> <td><code>yarn.lock</code></td> @@ -236,8 +243,8 @@ table.supported-languages ul { <td>C#</td> </tr> <tr> - <td rowspan="3">Python</td> - <td rowspan="3">3.6</td> + <td rowspan="4">Python</td> + <td rowspan="4">3.9</td> <td><a href="https://setuptools.readthedocs.io/en/latest/">setuptools</a></td> <td><code>setup.py</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -267,6 +274,12 @@ table.supported-languages ul { <td>N</td> </tr> <tr> + <td><a href="https://python-poetry.org/">Poetry</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-4">4</a></b></sup></td> + <td><code>poetry.lock</code></td> + <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> + <td>N</td> + </tr> + <tr> <td>Scala</td> <td>N/A</td> <td><a href="https://www.scala-sbt.org/">sbt</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-3">3</a></b></sup></td> @@ -305,6 +318,14 @@ table.supported-languages ul { Support for <a href="https://www.scala-sbt.org/">sbt</a> 1.3 and above was added in GitLab 13.9. </p> </li> + <li> + <a id="notes-regarding-supported-languages-and-package-managers-4"></a> + <p> + Support for <a href="https://python-poetry.org/">Poetry</a> projects with a <code>poetry.lock</code> file was <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/7006">added in GitLab 15.0</a>. + Support for projects without a <code>poetry.lock</code> file is tracked in issue: + <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/32774">Poetry's pyproject.toml support for dependency scanning.</a> + </p> + </li> </ol> <!-- markdownlint-enable MD044 --> @@ -321,13 +342,14 @@ The following package managers use lockfiles that GitLab analyzers are capable o | Package Manager | Supported File Format Versions | Tested Versions | | ------ | ------ | ------ | -| Bundler | N/A | [1.17.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/ruby-bundler/main/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | -| Composer | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/php-composer/main/composer.lock) | -| Conan | 0.4 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/c-conan/main/conan.lock) | -| Go | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/go-modules/main/go.sum) | -| NuGet | v1 | [4.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/csharp-nuget-dotnetcore/main/src/web.api/packages.lock.json#L2) | -| npm | v1, v2 | [6.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/main/package-lock.json#L4), [7.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/lockfileVersion2/package-lock.json#L4) | -| yarn | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/main/yarn.lock#L2) | +| Bundler | N/A | [1.17.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/ruby-bundler/default/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | +| Composer | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/php-composer/default/composer.lock) | +| Conan | 0.4 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/c-conan/default/conan.lock) | +| Go | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/go-modules/default/go.sum) | +| NuGet | v1 | [4.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/csharp-nuget-dotnetcore/default/src/web.api/packages.lock.json#L2) | +| npm | v1, v2 | [6.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/default/package-lock.json#L4), [7.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/lockfileVersion2/package-lock.json#L4) | +| yarn | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/default/yarn.lock#L2) | +| Poetry | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v3/qa/fixtures/python-poetry/default/poetry.lock) | #### Obtaining dependency information by running a package manager to generate a parsable file @@ -338,25 +360,18 @@ To support the following package managers, the GitLab analyzers proceed in two s | Package Manager | Pre-installed Versions | Tested Versions | | ------ | ------ | ------ | -| Bundler | [2.1.4](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit/-/blob/v2.11.3/Dockerfile#L15)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup> | [1.17.3](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/master/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | | sbt | [1.6.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.24.6/config/.tool-versions#L4) | [1.0.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L443-447), [1.1.6](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L449-453), [1.2.8](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L455-459), [1.3.12](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L461-465), [1.4.6](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L467-471), [1.5.8](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L473-477), [1.6.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L479-483) | | Maven | [3.6.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L95-97) | [3.6.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L95-97) | -| Gradle | [6.7.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.23.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup>, [7.3.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.26.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup> | [5.6.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L319-323), [6.7](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L286-288)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup>, [6.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L331-335), [7.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L300-302)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup> | +| Gradle | [6.7.1](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.23.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup>, [7.3.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.26.0/config/.tool-versions#L5)<sup><b><a href="#exported-dependency-information-notes-1">1</a></b></sup> | [5.6.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L319-323), [6.7](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L286-288)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup>, [6.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L331-335), [7.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/blob/v2.28.1/spec/image_spec.rb#L300-302)<sup><b><a href="#exported-dependency-information-notes-2">2</a></b></sup> | | setuptools | [50.3.2](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/v2.29.9/Dockerfile#L27) | [57.5.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L224-247) | | pip | [20.2.4](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/v2.29.9/Dockerfile#L26) | [20.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L77-91) | -| Pipenv | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.18.4/requirements.txt#L13) | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L168-191)<sup><b><a href="#exported-dependency-information-notes-4">4</a></b></sup>, [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L143-166) | +| Pipenv | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.18.4/requirements.txt#L13) | [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L168-191)<sup><b><a href="#exported-dependency-information-notes-3">3</a></b></sup>, [2018.11.26](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/blob/v2.22.0/spec/image_spec.rb#L143-166) | <!-- markdownlint-disable MD044 --> <ol> <li> <a id="exported-dependency-information-notes-1"></a> <p> - The pre-installed and tested version of <code>Bundler</code> is only used for the <a href="https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit">bundler-audit</a> analyzer, and is not used for <a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">gemnasium</a>. - </p> - </li> - <li> - <a id="exported-dependency-information-notes-2"></a> - <p> Different versions of Java require different versions of Gradle. The versions of Gradle listed in the above table are pre-installed in the analyzer image. The version of Gradle used by the analyzer depends on whether your project uses a <code>gradlew</code> (Gradle wrapper) file or not: @@ -383,13 +398,13 @@ To support the following package managers, the GitLab analyzers proceed in two s </ul> </li> <li> - <a id="exported-dependency-information-notes-3"></a> + <a id="exported-dependency-information-notes-2"></a> <p> These tests confirm that if a <code>gradlew</code> file does not exist, the version of <code>Gradle</code> pre-installed in the analyzer image is used. </p> </li> <li> - <a id="exported-dependency-information-notes-4"></a> + <a id="exported-dependency-information-notes-3"></a> <p> This test confirms that if a <code>Pipfile.lock</code> file is found, it will be used by <a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a> to scan the exact package versions listed in this file. </p> @@ -411,35 +426,28 @@ NOTE: If you've run into problems while scanning multiple files, please contribute a comment to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337056). -#### Ruby - -The following analyzers are executed, each of which have different behavior when processing multiple files: - -- [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) - - Supports multiple lockfiles. - -- [bundler-audit](https://github.com/rubysec/bundler-audit) - - Does not support multiple lockfiles. When multiple lockfiles exist, `bundler-audit` - analyzes the first lockfile discovered while traversing the directory tree in alphabetical order. +#### Python -WARNING: -The `bundler-audit` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#bundler-audit-dependency-scanning-tool). +We only execute one installation in the directory where either a requirements file or a lock file has been detected. Dependencies are only analyzed by `gemnasium-python` for the first file that is detected. Files are searched for in the following order: -#### Python +1. `requirements.txt`, `requirements.pip`, or `requires.txt` for projects using Pip. +1. `Pipfile` or `Pipfile.lock` for projects using Pipenv. +1. `poetry.lock` for projects using Poetry. +1. `setup.py` for project using Setuptools. -We only execute one installation in the directory where a requirements file has been detected, such as `requirements.txt` or any -variation of this file (for example, `requirements.pip` or `requires.txt`). +The search begins with the root directory and then continues with subdirectories if no builds are found in the root directory. Consequently a Poetry lock file in the root directory would be detected before a Pipenv file in a subdirectory. #### Java and Scala -We only execute one build in the directory where a build file has been detected, such as `build.sbt` or `build.gradle`. -Please note, we support the following types of Java project structures: +We only execute one build in the directory where a build file has been detected. For large projects that include +multiple Gradle, Maven, or sbt builds, or any combination of these, `gemnasium-maven` only analyzes dependencies for the first build file +that is detected. Build files are searched for in the following order: + +1. `build.gradle` or `build.gradle.kts` for single or [multi-project](https://docs.gradle.org/current/userguide/intro_multi_project_builds.html) Gradle builds. +1. `pom.xml` for single or [multi-module](https://maven.apache.org/pom.html#Aggregation) Maven projects. +1. `build.sbt` for single or [multi-project](https://www.scala-sbt.org/1.x/docs/Multi-Project.html) sbt builds. -- [multi-project sbt builds](https://www.scala-sbt.org/1.x/docs/Multi-Project.html) -- [multi-project Gradle builds](https://docs.gradle.org/current/userguide/intro_multi_project_builds.html) -- [multi-module maven projects](https://maven.apache.org/pom.html#Aggregation) +The search begins with the root directory and then continues with subdirectories if no builds are found in the root directory. Consequently an sbt build file in the root directory would be detected before a Gradle build file in a subdirectory. #### JavaScript @@ -454,26 +462,20 @@ The following analyzers are executed, each of which have different behavior when Does not support multiple lockfiles. When multiple lockfiles exist, `Retire.js` analyzes the first lockfile discovered while traversing the directory tree in alphabetical order. -From GitLab 14.8 the `Gemnasium` analyzer scans supported JavaScript projects for vendored libraries +From GitLab 14.8 the `gemnasium` analyzer scans supported JavaScript projects for vendored libraries (that is, those checked into the project but not managed by the package manager). -WARNING: -The `retire.js` analyzer is deprecated and will be removed in GitLab 15.0 since it duplicates the functionality of the `gemnasium` analyzer. For more information, read the [deprecation announcement](../../../update/deprecations.md#retire-js-dependency-scanning-tool). -We execute both analyzers because they use different sources of vulnerability data. The result is more comprehensive analysis than if only one was executed. - -#### PHP, Go, C, C++, .NET, C# +#### PHP, Go, C, C++, .NET, C#, Ruby, JavaScript The analyzer for these languages supports multiple lockfiles. -### Support for additional languages +#### Support for additional languages Support for additional languages, dependency managers, and dependency files are tracked in the following issues: | Package Managers | Languages | Supported files | Scan tools | Issue | | ------------------- | --------- | --------------- | ---------- | ----- | -| [Poetry](https://python-poetry.org/) | Python | `poetry.lock` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) | [GitLab#7006](https://gitlab.com/gitlab-org/gitlab/-/issues/7006) | - -For workarounds, see the [Troubleshooting section](#troubleshooting). +| [Poetry](https://python-poetry.org/) | Python | `pyproject.toml` | [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) | [GitLab#32774](https://gitlab.com/gitlab-org/gitlab/-/issues/32774) | ## Contribute your scanner @@ -506,7 +508,7 @@ always take the latest dependency scanning artifact available. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4908) in GitLab 14.1 [with a flag](../../../administration/feature_flags.md) named `sec_dependency_scanning_ui_enable`. Enabled by default. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/282533) in GitLab 14.1. -> - [Feature flag sec_dependency_scanning_ui_enable removed](https://gitlab.com/gitlab-org/gitlab/-/issues/326005) in GitLab 14.2. +> - [Feature flag `sec_dependency_scanning_ui_enable` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/326005) in GitLab 14.2. To enable Dependency Scanning in a project, you can create a merge request: @@ -577,9 +579,9 @@ The following variables allow configuration of global dependency scanning settin | ----------------------------|------------ | | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. The bundle of certificates provided here is also used by other tools during the scanning process, such as `git`, `yarn`, or `npm`. See [Using a custom SSL CA certificate authority](#using-a-custom-ssl-ca-certificate-authority) for more details. | | `DS_EXCLUDED_ANALYZERS` | Specify the analyzers (by name) to exclude from Dependency Scanning. For more information, see [Dependency Scanning Analyzers](analyzers.md). | -| `DS_DEFAULT_ANALYZERS` | ([**DEPRECATED - use `DS_EXCLUDED_ANALYZERS` instead**](https://gitlab.com/gitlab-org/gitlab/-/issues/287691)) Override the names of the official default images. For more information, see [Dependency Scanning Analyzers](analyzers.md). | +| `DS_DEFAULT_ANALYZERS` | This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in 15.0. Use `DS_EXCLUDED_ANALYZERS` instead. | | `DS_EXCLUDED_PATHS` | Exclude files and directories from the scan based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"`. | -| `DS_IMAGE_SUFFIX` | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354796) in GitLab 14.10. | +| `DS_IMAGE_SUFFIX` | Suffix added to the image name. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354796) in GitLab 14.10.) Automatically set to `"-fips"` when FIPS mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357922) in GitLab 15.0.) | | `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | | `SECURE_LOG_LEVEL` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. Default: `info`. | @@ -589,16 +591,13 @@ The following variables are used for configuring specific analyzers (used for a | CI/CD variable | Analyzer | Default | Description | |--------------------------------------| ------------------ | ---------------------------- |------------ | -| `BUNDLER_AUDIT_UPDATE_DISABLED` | `bundler-audit` | `"false"` | Disable automatic updates for the `bundler-audit` analyzer. Use if you're running dependency scanning in an offline, air-gapped environment.| -| `BUNDLER_AUDIT_ADVISORY_DB_URL` | `bundler-audit` | `https://github.com/rubysec/ruby-advisory-db` | URL of the advisory database used by bundler-audit. | -| `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME` | `bundler-audit` | `master` | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL`. | | `GEMNASIUM_DB_LOCAL_PATH` | `gemnasium` | `/gemnasium-db` | Path to local Gemnasium database. | | `GEMNASIUM_DB_UPDATE_DISABLED` | `gemnasium` | `"false"` | Disable automatic updates for the `gemnasium-db` advisory database (For usage see: [examples](#hosting-a-copy-of-the-gemnasium_db-advisory-database))| | `GEMNASIUM_DB_REMOTE_URL` | `gemnasium` | `https://gitlab.com/gitlab-org/security-products/gemnasium-db.git` | Repository URL for fetching the Gemnasium database. | | `GEMNASIUM_DB_REF_NAME` | `gemnasium` | `master` | Branch name for remote repository database. `GEMNASIUM_DB_REMOTE_URL` is required. | | `DS_REMEDIATE` | `gemnasium` | `"true"` | Enable automatic remediation of vulnerable dependencies. | | `GEMNASIUM_LIBRARY_SCAN_ENABLED` | `gemnasium` | `"true"` | Enable detecting vulnerabilities in vendored JavaScript libraries. For now, `gemnasium` leverages [`Retire.js`](https://github.com/RetireJS/retire.js) to do this job. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350512) in GitLab 14.8. | -| `DS_JAVA_VERSION` | `gemnasium-maven` | `11` | Version of Java. Available versions: `8`, `11`, `13`, `14`, `15`, `16`, `17`. | +| `DS_JAVA_VERSION` | `gemnasium-maven` | `17` | Version of Java. Available versions: `8`, `11`, `13`, `14`, `15`, `16`, `17`. | | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that are passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repositories). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that are passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer passes to `sbt`. | @@ -607,9 +606,6 @@ The following variables are used for configuring specific analyzers (used for a | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | -| `RETIREJS_JS_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository.json` | Path or URL to `retire.js` JS vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | -| `RETIREJS_NODE_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/npmrepository.json` | Path or URL to `retire.js` node vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | -| `RETIREJS_ADVISORY_DB_INSECURE` | `retire.js` | `false` | Enable fetching remote JS and Node vulnerability data files (defined by the `RETIREJS_JS_ADVISORY_DB` and `RETIREJS_NODE_ADVISORY_DB` variables) from hosts using an insecure or self-signed SSL (TLS) certificate. | #### Other variables @@ -667,32 +663,10 @@ Read more on [how to use private Maven repositories](../index.md#using-private-m GitLab also offers [FIPS-enabled Red Hat UBI](https://www.redhat.com/en/blog/introducing-red-hat-universal-base-image) versions of the Gemnasium images. You can therefore replace standard images with FIPS-enabled images. -To use FIPS-enabled images, set the `DS_IMAGE_SUFFIX` to `-fips`, -and set `DS_EXCLUDED_ANALYZERS` to `bundler-audit, retire.js` -to exclude the analyzers that don't support FIPS. +Gemnasium scanning jobs automatically use FIPS-enabled image when FIPS mode is enabled in the GitLab instance. +([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357922) in GitLab 15.0.) -```yaml -variables: - DS_IMAGE_SUFFIX: "-fips" - DS_EXCLUDED_ANALYZERS: "bundler-audit, retire.js" -``` - -If you want to execute `bundler-audit` or `retire.js` in your project pipeline, you can override the -Gemnasium scanning jobs, and set `DS_IMAGE_SUFFIX` to `-fips` only for those jobs. - -```yaml -gemnasium-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" - -gemnasium-maven-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" - -gemnasium-python-dependency_scanning: - variables: - DS_IMAGE_SUFFIX: "-fips" -``` +To manually switch to FIPS-enabled images, set the variable `DS_IMAGE_SUFFIX` to `"-fips"`. ## Interacting with the vulnerabilities @@ -944,9 +918,6 @@ Here are the requirements for using dependency scanning in an offline environmen This advisory database is constantly being updated, so you must periodically sync your local copy with GitLab. -- _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). -- _Only if scanning npm/yarn projects_: Host an offline copy of the [`retire.js`](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [`js`](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. - Note that GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), meaning the runner tries to pull Docker images from the GitLab container registry even if a local copy is available. The GitLab Runner [`pull_policy` can be set to `if-not-present`](https://docs.gitlab.com/runner/executors/docker.html#using-the-if-not-present-pull-policy) @@ -961,11 +932,9 @@ import the following default dependency scanning analyzer images from `registry. your [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/security-products/gemnasium:2 -registry.gitlab.com/security-products/gemnasium-maven:2 -registry.gitlab.com/security-products/gemnasium-python:2 -registry.gitlab.com/security-products/retire.js:2 -registry.gitlab.com/security-products/bundler-audit:2 +registry.gitlab.com/security-products/gemnasium:3 +registry.gitlab.com/security-products/gemnasium-maven:3 +registry.gitlab.com/security-products/gemnasium-python:3 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -987,8 +956,6 @@ Support for custom certificate authorities was introduced in the following versi | `gemnasium` | [v2.8.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/releases/v2.8.0) | | `gemnasium-maven` | [v2.9.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven/-/releases/v2.9.0) | | `gemnasium-python` | [v2.7.0](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python/-/releases/v2.7.0) | -| `retire.js` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js/-/releases/v2.4.0) | -| `bundler-audit` | [v2.4.0](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit/-/releases/v2.4.0) | ### Set dependency scanning CI/CD job variables to use local dependency scanning analyzers @@ -1121,22 +1088,6 @@ intended to obtain a private package from a private index. This only affects use requires that the package does not already exist in the public index (and thus the attacker can put the package there with an arbitrary version number). -## Limitations - -### Referencing local dependencies using a path in JavaScript projects - -The [Retire.js](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) analyzer -doesn't support dependency references made with [local paths](https://docs.npmjs.com/cli/v6/configuring-npm/package-json/#local-paths) -in the `package.json` of JavaScript projects. The dependency scan outputs the following error for -such references: - -```plaintext -ERROR: Could not find dependencies: <dependency-name>. You may need to run npm install -``` - -As a workaround, add the [`retire.js`](analyzers.md) analyzer to -[`DS_EXCLUDED_ANALYZERS`](#configuring-dependency-scanning). - ## Troubleshooting ### Working around missing support for certain languages or package managers @@ -1156,9 +1107,8 @@ Generally, the approach is the following: 1. Add [`dependencies: [<your-converter-job>]`](../../../ci/yaml/index.md#dependencies) to your `dependency_scanning` job to make use of the converted definitions files. -For example, the unsupported `poetry.lock` file can be -[converted](https://python-poetry.org/docs/cli/#export) -to the supported `requirements.txt` as follows. +For example, Poetry projects that _only_ have a `pyproject.toml` +file can generate the `poetry.lock` file as follows. ```yaml include: @@ -1167,15 +1117,11 @@ include: stages: - test -variables: - PIP_REQUIREMENTS_FILE: "requirements-converted.txt" - gemnasium-python-dependency_scanning: - # Work around https://gitlab.com/gitlab-org/gitlab/-/issues/7006 + # Work around https://gitlab.com/gitlab-org/gitlab/-/issues/32774 before_script: - - pip install poetry # Or via another method: https://python-poetry.org/docs/#installation - - poetry export --output="$PIP_REQUIREMENTS_FILE" - - rm poetry.lock pyproject.toml + - pip install "poetry>=1,<2" # Or via another method: https://python-poetry.org/docs/#installation + - poetry update --lock # Generates the lock file to be analyzed. ``` ### `Error response from daemon: error processing tar file: docker-tar: relocation error` @@ -1197,11 +1143,6 @@ syntax. This directive is limited to 10000 checks and always returns `true` afte number. Because of this, and depending on the number of files in your repository, a dependency scanning job might be triggered even if the scanner doesn't support your project. -### Issues building projects with npm or yarn packages relying on Python 2 - -[Python 2 was removed](https://www.python.org/doc/sunset-python-2/) from the `retire.js` analyzer in GitLab 13.7 (analyzer version 2.10.1). Projects using packages -with a dependency on this version of Python should use `retire.js` version 2.10.0 or lower (for example, `registry.gitlab.com/gitlab-org/security-products/analyzers/retire.js:2.10.0`). - ### Error: `dependency_scanning is used for configuration only, and its script should not be executed` For information on this, see the [GitLab Secure troubleshooting section](../index.md#error-job-is-used-for-configuration-only-and-its-script-should-not-be-executed). @@ -1260,7 +1201,7 @@ We recommend committing the lock files, which prevents this warning. If you have manually set `DS_MAJOR_VERSION` or `DS_ANALYZER_IMAGE` for specific reasons, and now must update your configuration to again get the latest patched versions of our -analyzers, edit your `gitlab-ci.yml` file and either: +analyzers, edit your `.gitlab-ci.yml` file and either: - Set your `DS_MAJOR_VERSION` to match the latest version as seen in [our current Dependency Scanning template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml#L18). @@ -1290,5 +1231,22 @@ To work around this error, downgrade the analyzer's version of `setuptools` (e.g gemnasium-python-dependency_scanning: before_script: - pip install setuptools==57.5.0 - image: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 +``` + +### Dependency Scanning of projects using psycopg2 fails with `pg_config executable not found` error + +Scanning a Python project that depends on `psycopg2` can fail with this message: + +```plaintext +Error: pg_config executable not found. +``` + +[psycopg2](https://pypi.org/project/psycopg2/) depends on the `libpq-dev` Debian package, +which is not installed in the `gemnasium-python` Docker image. To work around this error, +install the `libpq-dev` package in a `before_script`: + +```yaml +gemnasium-python-dependency_scanning: + before_script: + - apt-get update && apt-get install -y libpq-dev ``` diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 3a6aa8e3485..b5d39f3b32a 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -10,7 +10,7 @@ GitLab can check your application for security vulnerabilities including: - Unauthorized access. - Data leaks. -- Denial of service attacks. +- Denial of Service (DoS) attacks. Statistics and details on vulnerabilities are included in the merge request. Providing actionable information _before_ changes are merged enables you to be proactive. @@ -19,9 +19,6 @@ GitLab also provides high-level statistics of vulnerabilities across projects an - The [Security Dashboard](security_dashboard/index.md) provides a high-level view of vulnerabilities detected in your projects, pipeline, and groups. -- The [Threat Monitoring](threat_monitoring/index.md) page provides runtime security metrics - for application environments. With the information provided, - you can immediately begin risk analysis and remediation. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview of GitLab application security, see [Shifting Security Left](https://www.youtube.com/watch?v=XnYstHObqlA&t). @@ -51,8 +48,8 @@ The following vulnerability scanners and their databases are regularly updated: | Secure scanning tool | Vulnerabilities database updates | |:----------------------------------------------------------------|:---------------------------------| -| [Container Scanning](container_scanning/index.md) | A job runs on a daily basis to build new images with the latest vulnerability database updates from the upstream scanner. For more details, see [Vulnerabilities database update](container_scanning/index.md#vulnerabilities-database-update). | -| [Dependency Scanning](dependency_scanning/index.md) | Relies on `bundler-audit` (for Ruby gems), `retire.js` (for npm packages), and `gemnasium` (the GitLab tool for all libraries). Both `bundler-audit` and `retire.js` fetch their vulnerabilities data from GitHub repositories, so vulnerabilities added to `ruby-advisory-db` and `retire.js` are immediately available. The tools themselves are updated once per month if there's a new version. The [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db) is updated on a daily basis using [data from NVD, the `ruby-advisory-db` and the GitHub Advisory Database as data sources](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/master/SOURCES.md). See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | +| [Container Scanning](container_scanning/index.md) | A job runs on a daily basis to build new images with the latest vulnerability database updates from the upstream scanner. For more details, see [Vulnerabilities database update](container_scanning/index.md#vulnerabilities-database). | +| [Dependency Scanning](dependency_scanning/index.md) | Relies on the [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db). It is updated on a daily basis using [data from NVD, the `ruby-advisory-db` and the GitHub Advisory Database as data sources](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/master/SOURCES.md). See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | | [Dynamic Application Security Testing (DAST)](dast/index.md) | The scanning engine is updated on a periodic basis. See the [version of the underlying tool `zaproxy`](https://gitlab.com/gitlab-org/security-products/dast/blob/main/Dockerfile#L1). The scanning rules are downloaded at scan runtime. | | [Static Application Security Testing (SAST)](sast/index.md) | Relies exclusively on [the tools GitLab wraps](sast/index.md#supported-languages-and-frameworks). The underlying analyzers are updated at least once per month if a relevant update is available. The vulnerabilities database is updated by the upstream tools. | @@ -145,7 +142,7 @@ Jobs pass if they are able to complete a scan. A _pass_ result does NOT indicate Jobs fail if they are unable to complete a scan. You can view the pipeline logs for more information. -All jobs are permitted to fail by default. This means that if they fail it do not fail the pipeline. +All jobs are permitted to fail by default. This means that if they fail, it does not fail the pipeline. If you want to prevent vulnerabilities from being merged, you should do this by adding [Security Approvals in Merge Requests](#security-approvals-in-merge-requests) which prevents unknown, high or critical findings from being merged without an approval from a specific group of people that you choose. @@ -174,7 +171,7 @@ reports are available to download. To download a report, select A merge request contains a security widget which displays a summary of the NEW results. New results are determined by comparing the current findings against existing findings in the target (default) branch (if there are prior findings). -We recommended you run a scan of the `default` branch before enabling feature branch scans for your developers. Otherwise, there is no base for comparison and all feature branches display the full scan results in the merge request security widget. +We recommend you run a scan of the `default` branch before enabling feature branch scans for your developers. Otherwise, there is no base for comparison and all feature branches display the full scan results in the merge request security widget. The merge request security widget displays only a subset of the vulnerabilities in the generated JSON artifact because it contains both NEW and EXISTING findings. @@ -204,56 +201,24 @@ By default, the vulnerability report does not show vulnerabilities of `dismissed ## Security approvals in merge requests -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9928) in GitLab 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9928) in GitLab 12.2. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. You can enforce an additional approval for merge requests that would introduce one of the following security issues: -- A security vulnerability. For more details, read - [Vulnerability-Check rule](#vulnerability-check-rule). +- A security vulnerability. For more details, read [Scan result policies](policies/scan-result-policies.md). - A software license compliance violation. For more details, read [Enabling license approvals within a project](../compliance/license_compliance/index.md#enabling-license-approvals-within-a-project). -### Vulnerability-Check rule +### Migration of existing Vulnerability-Check rules -WARNING: -This feature is in its end-of-life process. It is [deprecated](../../update/deprecations.md#vulnerability-check) -in GitLab 14.8, and is planned for removal in GitLab 15.0. Users should migrate to the new -[Security Approval Policies](policies/scan-result-policies.md). +If your projects have rules that have a security orchestration project, a new MR with +the existing rule's content is created automatically against the default branch belonging +to the security orchestration project. To maintain the same security approval rules you +had before GitLab 15.0, we recommend merging this new MR. -To prevent a merge request introducing a security vulnerability in a project, enable the -Vulnerability-Check rule. While this rule is enabled, additional merge request approval by -[eligible approvers](../project/merge_requests/approvals/rules.md#eligible-approvers) -is required when the latest security report in a merge request: - -- Contains vulnerabilities with states (for example, `previously detected`, `dismissed`) matching the rule's vulnerability states. Only `newly detected` are considered if the target branch differs from the project default branch. -- Contains vulnerabilities with severity levels (for example, `high`, `critical`, or `unknown`) - matching the rule's severity levels. -- Contains a vulnerability count higher than the rule allows. -- Is not yet generated (until pipeline completion). - -An approval is optional when the security report: - -- Contains only vulnerabilities with states (for example, `newly detected`, `resolved`) **NOT** matching the rule's vulnerability states. -- Contains only vulnerabilities with severity levels (for example, `low`, `medium`) **NOT** matching - the rule's severity levels. -- Contains a vulnerability count equal to or less than what the rule allows. - -Project members with at least the Maintainer role can enable or edit -the Vulnerability-Check rule. - -#### Enable the Vulnerability-Check rule - -To enable or edit the Vulnerability-Check rule: - -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > General**. -1. Expand **Merge request approvals**. -1. Select **Activate** or **Edit** of the Vulnerability-Check. -1. Complete the fields. **Approvals required** must be at least 1. -1. Select **Add approval rule**. - -The approval rule is enabled for all merge requests. Any code changes reset the approvals required. +If your projects have rules without a security orchestration project, a new security orchestration project is created automatically with the content of the existing rule. No additional action is required. ## Using private Maven repositories @@ -443,7 +408,7 @@ You can interact with the results of the security scanning tools in several loca For more details about which findings or vulnerabilities you can view in each of those locations, select the respective link. Each page details the ways in which you can interact with the findings -and vulnerabilities. As an example, in most cases findings start out as _detected_ status. +and vulnerabilities. As an example, in most cases findings start out as a _detected_ status. You have the option to: @@ -507,7 +472,7 @@ Additional details about the differences between the two solutions are outlined | ------ | ------ | ------ | | **Flexibility** | Supports anything that can be done in a CI file. | Limited to only the items for which GitLab has explicitly added support. DAST, SAST, Secret Detection, and Container Scanning scans are supported. | | **Usability** | Requires knowledge of CI YAML. | Follows a `rules` and `actions`-based YAML structure. | -| **Inclusion in CI pipeline** | The compliance pipeline is executed instead of the project's `gitlab-ci.yml` file. To include the project's `gitlab-ci.yml` file, use an `include` statement. Defined variables aren't allowed to be overwritten by the included project's YAML file. | Forced inclusion of a new job into the CI pipeline. DAST jobs that must be customized on a per-project basis can have project-level Site Profiles and Scan Profiles defined. To ensure separation of duties, these profiles are immutable when referenced in a scan execution policy. All jobs can be customized as part of the security policy itself with the same variables that are normally available to the CI job. | +| **Inclusion in CI pipeline** | The compliance pipeline is executed instead of the project's `.gitlab-ci.yml` file. To include the project's `.gitlab-ci.yml` file, use an `include` statement. Defined variables aren't allowed to be overwritten by the included project's YAML file. | Forced inclusion of a new job into the CI pipeline. DAST jobs that must be customized on a per-project basis can have project-level Site Profiles and Scan Profiles defined. To ensure separation of duties, these profiles are immutable when referenced in a scan execution policy. All jobs can be customized as part of the security policy itself with the same variables that are normally available to the CI job. | | **Schedulable** | Can be scheduled through a scheduled pipeline on the group. | Can be scheduled natively through the policy configuration itself. | | **Separation of Duties** | Only group owners can create compliance framework labels. Only project owners can apply compliance framework labels to projects. The ability to make or approve changes to the compliance pipeline definition is limited to individuals who are explicitly given access to the project that contains the compliance pipeline. | Only project owners can define a linked security policy project. The ability to make or approve changes to security policies is limited to individuals who are explicitly given access to the security policy project. | | **Ability to apply one standard to multiple projects** | The same compliance framework label can be applied to multiple projects inside a group. | The same security policy project can be used for multiple projects across GitLab with no requirement of being located in the same group. | @@ -657,7 +622,7 @@ involve pinning to the previous template versions, for example: ``` Additionally, we provide a dedicated project containing the versioned legacy templates. -This can be used for offline setups or anyone wishing to use [Auto DevOps](../../topics/autodevops/index.md). +This can be used for offline setups or for anyone wishing to use [Auto DevOps](../../topics/autodevops/index.md). Instructions are available in the [legacy template project](https://gitlab.com/gitlab-org/auto-devops-v12-10). @@ -694,3 +659,6 @@ These security pages can be populated by running the jobs from the manual step o There is [an issue open to handle this scenario](https://gitlab.com/gitlab-org/gitlab/-/issues/346843). Please upvote the issue to help with prioritization, and [contributions are welcomed](https://about.gitlab.com/community/contribute/). + doc/user/project/merge_requests/approvals/settings.md ++ +0 diff --git a/doc/user/application_security/policies/img/policies_list_v14_3.png b/doc/user/application_security/policies/img/policies_list_v14_3.png Binary files differdeleted file mode 100644 index 7a24860d4a7..00000000000 --- a/doc/user/application_security/policies/img/policies_list_v14_3.png +++ /dev/null diff --git a/doc/user/application_security/policies/img/policies_list_v15_0.png b/doc/user/application_security/policies/img/policies_list_v15_0.png Binary files differnew file mode 100644 index 00000000000..4089c311fe4 --- /dev/null +++ b/doc/user/application_security/policies/img/policies_list_v15_0.png diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index 81d24104340..f790164d9a0 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -19,7 +19,6 @@ GitLab supports the following security policies: - [Scan Execution Policy](scan-execution-policies.md) - [Scan Result Policy](scan-result-policies.md) -- [Container Network Policy](#container-network-policy) (DEPRECATED) ## Security policy project @@ -42,8 +41,9 @@ stored there. Examples and schema information are available for the following po - [Scan execution policy](scan-execution-policies.md#example-security-policies-project) - [Scan result policy](scan-result-policies.md#example-security-scan-result-policies-project) -Policies created in this project are applied through a background job that runs once every 10 -minutes. Allow up to 10 minutes for any policy changes committed to this project to take effect. +Most policy changes take effect as soon as the merge request is merged. Any changes that +do not go through a merge request and are committed directly to the default branch may require up to 10 minutes +before the policy changes take effect. ### Managing the linked security policy project @@ -81,22 +81,7 @@ status), and create and edit deployed policies: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Policies**. - - -Network policies are fetched directly from the selected environment's -deployment platform while other policies are fetched from the project's -security policy project. Changes performed outside of this tab are -reflected upon refresh. - -By default, the policy list contains predefined network policies in a -disabled state. Once enabled, a predefined policy deploys to the -selected environment's deployment platform and you can manage it like -the regular policies. - -Note that if you're using [Auto DevOps](../../../topics/autodevops/index.md) -and change a policy in this section, your `auto-deploy-values.yaml` file doesn't update. Auto DevOps -users must make changes by following the -[Container Network Policy documentation](../../../topics/autodevops/stages.md#network-policy). + ## Policy editor @@ -144,111 +129,6 @@ See [Scan execution policies](scan-execution-policies.md). See [Scan result policies](scan-result-policies.md). -## Container Network Policy - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32365) in GitLab 12.9. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Container Network Policy is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The **Container Network Policy** section provides packet flow metrics for -your application's Kubernetes namespace. This section has the following -prerequisites: - -- Your project contains at least one [environment](../../../ci/environments/index.md). -- You've [installed Cilium](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). -- You've configured the [Prometheus service](../../project/integrations/prometheus.md#enabling-prometheus-integration). - -If you're using custom Helm values for Cilium, you must enable Hubble -with flow metrics for each namespace by adding the following lines to -your [Cilium values](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium): - -```yaml -hubble: - enabled: true - metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' -``` - -The **Container Network Policy** section displays the following information -about your packet flow: - -- The total amount of the inbound and outbound packets -- The proportion of packets dropped according to the configured - policies -- The per-second average rate of the forwarded and dropped packets - accumulated over time window for the requested time interval - -If a significant percentage of packets is dropped, you should -investigate it for potential threats by -examining the Cilium logs: - -```shell -kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor -``` - -### Change the status - -To change a network policy's status: - -- Select the network policy you want to update. -- Select **Edit policy**. -- Select the **Policy status** toggle to update the selected policy. -- Select **Save changes** to deploy network policy changes. - -Disabled network policies have the `network-policy.gitlab.com/disabled_by: gitlab` selector inside -the `podSelector` block. This narrows the scope of such a policy and as a result it doesn't affect -any pods. The policy itself is still deployed to the corresponding deployment namespace. - -### Container Network Policy editor - -The policy editor only supports the [CiliumNetworkPolicy](https://docs.cilium.io/en/v1.8/policy/) -specification. Regular Kubernetes [NetworkPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#networkpolicy-v1-networking-k8s-io) -resources aren't supported. - -Rule mode supports the following rule types: - -- [Labels](https://docs.cilium.io/en/v1.8/policy/language/#labels-based). -- [Entities](https://docs.cilium.io/en/v1.8/policy/language/#entities-based). -- [IP/CIDR](https://docs.cilium.io/en/v1.8/policy/language/#ip-cidr-based). Only - the `toCIDR` block without `except` is supported. -- [DNS](https://docs.cilium.io/en/v1.8/policy/language/#dns-based). -- [Level 4](https://docs.cilium.io/en/v1.8/policy/language/#layer-4-examples) - can be added to all other rules. - -Once your policy is complete, save it by selecting **Save policy** -at the bottom of the editor. Existing policies can also be -removed from the editor interface by selecting **Delete policy** -at the bottom of the editor. - -### Configure a Network Policy Alert - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3438) and [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/287676) in GitLab 13.9. -> - The feature flag was removed and the Threat Monitoring Alerts Project was [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/287676) in GitLab 14.0. - -You can use policy alerts to track your policy's impact. Alerts are only available if you've -[installed](../../clusters/agent/repository.md) -and [configured](../../clusters/agent/install/index.md#register-the-agent-with-gitlab) -an agent for this project. - -There are two ways to create policy alerts: - -- In the [policy editor UI](#container-network-policy-editor), - by clicking **Add alert**. -- In the policy editor's YAML mode, through the `metadata.annotations` property: - - ```yaml - metadata: - annotations: - app.gitlab.com/alert: 'true' - ``` - -Once added, the UI updates and displays a warning about the dangers of too many alerts. - ## Roadmap See the [Category Direction page](https://about.gitlab.com/direction/protect/security_orchestration/) diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md index 7e8e60768b9..aa23ad30a73 100644 --- a/doc/user/application_security/policies/scan-execution-policies.md +++ b/doc/user/application_security/policies/scan-execution-policies.md @@ -35,8 +35,9 @@ policy project is automatically created. Existing policies can also be removed from the editor interface by selecting **Delete policy** at the bottom of the editor. -All scan execution policy changes are applied through a background job that runs once every 10 -minutes. Allow up to 10 minutes for any policy changes committed to this project to take effect. +Most policy changes take effect as soon as the merge request is merged. Any changes that +do not go through a merge request and are committed directly to the default branch may require up to 10 minutes +before the policy changes take effect.  @@ -84,9 +85,31 @@ This rule enforces the defined actions and schedules a scan on the provided date | `type` | `string` | `schedule` | The rule's type. | | `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). | | `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. | -| `clusters` | `object` | | The cluster where the given policy enforces running selected scans (only for `container_scanning`/`cluster_image_scanning` scans). The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. | +| `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [cluster image scanning](../../clusters/agent/vulnerabilities.md) will run. The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. | <!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> +| `clusters` (removed) | `object` | | This field was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in 15.0. Use the `agents` field instead. The cluster where the given policy enforces running selected scans (only for `container_scanning`/`cluster_image_scanning` scans). The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. | +<!--- end_remove --> -### `cluster` schema +GitLab supports the following types of CRON syntax for the `cadence` field: + +- A daily cadence of once per hour at a specified hour, for example: `0 18 * * *` +- A weekly cadence of once per week on a specified day and at a specified hour, for example: `0 13 * * 0` + +It is possible that other elements of the CRON syntax will work in the cadence field, however, GitLab does not officially test or support them. + +### `agent` schema + +Use this schema to define `agents` objects in the [`schedule` rule type](#schedule-rule-type). + +| Field | Type | Possible values | Description | +|--------------|---------------------|--------------------------|-------------| +| `namespaces` | `array` of `string` | | The namespace that is scanned. If empty, all namespaces will be scanned. | + +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> + +### `cluster` schema (removed) + +This schema was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in 15.0. +Use the [`agent` schema](#agent-schema) instead. Use this schema to define `clusters` objects in the [`schedule` rule type](#schedule-rule-type). @@ -97,6 +120,8 @@ Use this schema to define `clusters` objects in the [`schedule` rule type](#sche | `namespaces` | `array` of `string` | | The namespace that is scanned (only the first value is currently supported). | | `kinds` | `array` of `string` | `deployment`/`daemonset` | The resource kind that should be scanned (only the first value is currently supported). | +<!--- end_remove --> + ## `scan` action type This action executes the selected `scan` with additional parameters when conditions for at least one diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index d2cce207bfd..232a5c9f91c 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -9,7 +9,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can use scan result policies to take action based on scan results. For example, one type of scan result policy is a security approval policy that allows approval to be required based on the findings of one or more security scan jobs. Scan result policies are evaluated after a CI scanning -job is fully executed. +job is fully executed. The following video gives you an overview of GitLab scan result policies: + +<div class="video-fallback"> + See the video: <a href="https://youtu.be/w5I9gcUgr9U">Overview of GitLab Scan Result Policies</a>. +</div> +<figure class="video-container"> + <iframe src="https://www.youtube.com/embed/w5I9gcUgr9U" frameborder="0" allowfullscreen="true"> </iframe> +</figure> ## Scan result policy editor @@ -25,8 +32,9 @@ If a security policy project doesn't link to your project, GitLab creates such a Existing policies can also be removed from the editor interface by selecting **Delete policy** at the bottom of the editor. -All scan result policy changes are applied through a background job that runs once every 10 minutes. -Allow up to 10 minutes for any policy changes committed to this project to take effect. +Most policy changes take effect as soon as the merge request is merged. Any changes that +do not go through a merge request and are committed directly to the default branch may require up to 10 minutes +before the policy changes take effect. The [policy editor](index.md#policy-editor) supports YAML mode and rule mode. @@ -61,7 +69,7 @@ This rule enforces the defined actions based on the information provided. | Field | Type | Possible values | Description | |------------|------|-----------------|-------------| | `type` | `string` | `scan_finding` | The rule's type. | -| `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). | +| `branches` | `array` of `string` | `[]` or the branch's name | Protected branches for this rule to consider. | | `scanners` | `array` of `string` | `sast`, `secret_detection`, `dependency_scanning`, `container_scanning`, `dast`, `coverage_fuzzing`, `api_fuzzing` | The security scanners for this rule to consider. | | `vulnerabilities_allowed` | `integer` | Greater than or equal to zero | Number of vulnerabilities allowed before this rule is considered. | | `severity_levels` | `array` of `string` | `info`, `unknown`, `low`, `medium`, `high`, `critical`| The severity levels for this rule to consider. | diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 7529bf90ccf..661f564828a 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -4,20 +4,25 @@ group: Static Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# SAST Analyzers **(FREE)** +# SAST analyzers **(FREE)** > [Moved](https://gitlab.com/groups/gitlab-org/-/epics/2098) from GitLab Ultimate to GitLab Free in 13.3. -SAST relies on underlying third party tools that are wrapped into what we call -"Analyzers". An analyzer is a -[dedicated project](https://gitlab.com/gitlab-org/security-products/analyzers) -that wraps a particular tool to: +Static Application Security Testing (SAST) uses analyzers +to detect vulnerabilities in source code. Each analyzer is a wrapper around a [scanner](../terminology/#scanner), a third-party code analysis tool. -- Expose its detection logic. -- Handle its execution. -- Convert its output to the common format. +The analyzers are published as Docker images that SAST uses to launch dedicated containers for each +analysis. -This is achieved by implementing the [common API](https://gitlab.com/gitlab-org/security-products/analyzers/common). +SAST default images are maintained by GitLab, but you can also integrate your own custom image. + +For each scanner, an analyzer: + +- Exposes its detection logic. +- Handles its execution. +- Converts its output to a [standard format](../terminology/#secure-report-format). + +## SAST analyzers SAST supports the following official analyzers: @@ -36,12 +41,6 @@ SAST supports the following official analyzers: - [`sobelow`](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) (Sobelow (Elixir Phoenix)) - [`spotbugs`](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) (SpotBugs with the Find Sec Bugs plugin (Ant, Gradle and wrapper, Grails, Maven and wrapper, SBT)) -The analyzers are published as Docker images that SAST uses to launch -dedicated containers for each analysis. - -SAST is pre-configured with a set of **default images** that are maintained by -GitLab, but users can also integrate their own **custom images**. - ## SAST analyzer features For an analyzer to be considered Generally Available, it is expected to minimally @@ -55,34 +54,140 @@ support the following features: - [Emits JSON report format](index.md#reports-json-format) - [SELinux support](index.md#running-sast-in-selinux) -## Official default analyzers +## Post analyzers + +Post analyzers enrich the report output by an analyzer. A post analyzer doesn't modify report +content directly. Instead, it enhances the results with additional properties, including: + +- CWEs. +- Location tracking fields. +- A means of identifying false positives or insignificant findings. **(ULTIMATE)** + +## Data provided by analyzers + +Each analyzer provides data about the vulnerabilities it detects. The following table details the +data available from each analyzer. The values provided by these tools are heterogeneous so they are sometimes +normalized into common values, for example, `severity` and `confidence`. + +| Property / tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Semgrep | Sobelow | +|--------------------------------|------|--------|----------|-----------------|----------|------------|-------|-----------------|-------|------------|-----------------------|---------------------------|---------|---------| +| Affected item (for example, class or package) | ✓ | ✗ | ✓ | ✗ | ✓ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Confidence | ✗ | ✓ | ✓ | ✗ | ✓ | x | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✓ | +| Description | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | +| End column | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| End line | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| External ID (for example, CVE) | ✗ | ✗ | ⚠ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | +| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Internal doc/explanation | ✓ | ⚠ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | +| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | +| Severity | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ⚠ | ✗ | +| Solution | ✓ | ✗ | ✗ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | +| Source code extract | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Start column | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | +| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| URLs | ✓ | ✗ | ✓ | ✗ | ⚠ | ✗ | ⚠ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | + +- ✓ => Data is available. +- ⚠ => Data is available, but it's partially reliable, or it has to be extracted from unstructured content. +- ✗ => Data is not available or it would require specific, inefficient or unreliable, logic to obtain it. + +## Transition to Semgrep-based scanning + +SAST includes a [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) that covers [multiple languages](index.md#supported-languages-and-frameworks). +GitLab maintains the analyzer and writes detection rules for it. + +If you use the [GitLab-managed CI/CD template](index.md#configuration), the Semgrep-based analyzer operates alongside other language-specific analyzers. +It runs with GitLab-managed detection rules that mimic the other analyzers' detection rules. +Work to remove language-specific analyzers and replace them with the Semgrep-based analyzer is tracked in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/5245). + +You can choose to disable the other analyzers early and use Semgrep-based scanning for supported languages before the default behavior changes. If you do so: + +- You'll enjoy significantly faster scanning, reduced CI minutes usage, and more customizable scanning rules. +- However, vulnerabilities previously reported by language-specific analyzers will be reported again under certain conditions, including if you've dismissed the vulnerabilities before. The system behavior depends on: + - whether you've excluded the Semgrep-based analyzer from running in the past. + - which analyzer first discovered the vulnerabilities shown in the project's [Vulnerability Report](../vulnerability_report/). + +### Vulnerability translation + +When you switch analyzers for a language, vulnerabilities may not match up. + +The Vulnerability Management system automatically moves vulnerabilities from the old analyzer to Semgrep for certain languages: + +- For C, a vulnerability is moved if it has only ever been detected by Flawfinder in pipelines where Semgrep also detected it. Semgrep coverage for C was introduced by default into the CI/CD template in GitLab 14.4 (October 2021). +- For Go, a vulnerability is moved if it has only ever been detected by Gosec in pipelines where Semgrep also detected it. Semgrep coverage for Go was introduced by default into the CI/CD template in GitLab 14.2 (August 2021). +- For JavaScript and TypeScript, a vulnerability is moved if it has only ever been detected by ESLint in pipelines where Semgrep also detected it. Semgrep coverage for these languages was introduced into the CI/CD template in GitLab 13.12 (May 2021). + +However, you'll see old vulnerabilities re-created based on Semgrep results if: + +- A vulnerability was created by Bandit or SpotBugs and you disable those analyzers. We only recommend disabling Bandit and SpotBugs now if the analyzers aren’t working. Work to automatically translate Bandit and SpotBugs vulnerabilities to Semgrep is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328062). +- A vulnerability was created by ESLint, Gosec, or Flawfinder in a default-branch pipeline where Semgrep scanning did not run successfully (before Semgrep coverage was introduced for the language, because you disabled Semgrep explicitly, or because the Semgrep scan failed in that pipeline). We do not currently plan to combine these vulnerabilities if they already exist. -Any custom change to the official analyzers can be achieved by using a -[CI/CD variable in your `.gitlab-ci.yml`](index.md#available-cicd-variables). +When a vulnerability is re-created, the original vulnerability is marked as “no longer detected” in the Vulnerability Report. +A new vulnerability is then created based on the Semgrep finding. -### Using a custom Docker mirror +### Activating Semgrep-based scanning early -You can switch to a custom Docker registry that provides the official analyzer -images under a different prefix. For instance, the following instructs -SAST to pull `my-docker-registry/gl-images/sast/bandit` -instead of `registry.gitlab.com/security-products/sast/bandit`. -In `.gitlab-ci.yml` define: +You can choose to use Semgrep-based scanning instead of language-specific analyzers before the default behavior changes. + +We recommend taking this approach if any of these cases applies: + +- You haven't used SAST before on a project, so you don't already have SAST vulnerabilities in your [Vulnerability Report](../vulnerability_report/). +- You're having trouble configuring one of the analyzers whose coverage overlaps with Semgrep-based coverage. For example, you might have trouble setting up the SpotBugs-based analyzer to compile your code. +- You've already seen and dismissed vulnerabilities created by ESLint, Gosec, or Flawfinder scanning, and you've kept the re-created vulnerabilities created by Semgrep. + +You can make a separate choice for each of the language-specific analyzers, or you can disable them all. + +#### Activate Semgrep-based scanning + +To switch to Semgrep-based scanning early, you can: + +1. Create a merge request (MR) to set the [`SAST_EXCLUDED_ANALYZERS` CI/CD variable](#disable-specific-default-analyzers) to `"bandit,gosec,eslint"`. + - If you also want to disable SpotBugs scanning, add `spotbugs` to the list. We only recommend this for Java projects. SpotBugs is the only current analyzer that can scan Groovy, Kotlin, and Scala. + - If you also want to disable Flawfinder scanning, add `flawfinder` to the list. We only recommend this for C projects. Flawfinder is the only current analyzer that can scan C++. +1. Verify that scanning jobs succeed in the MR. You'll notice findings from the removed analyzers in _Fixed_ and findings from Semgrep in _New_. (Some findings may show different names, descriptions, and severities, since GitLab manages and edits the Semgrep rulesets.) +1. Merge the MR and wait for the default-branch pipeline to run. +1. Use the Vulnerability Report to dismiss the findings that are no longer detected by the language-specific analyzers. + +## Customize analyzers + +Use [CI/CD variables](index.md#available-cicd-variables) +in your `.gitlab-ci.yml` file to customize the behavior of your analyzers. + +### Use a custom Docker mirror + +You can use a custom Docker registry, instead of the GitLab registry, to host the analyzers' images. + +Prerequisites: + +- The custom Docker registry must provide images for all the official analyzers. + +NOTE: +This variable affects all Secure analyzers, not just the analyzers for SAST. + +To have GitLab download the analyzers' images from a custom Docker registry, define the prefix with +the `SECURE_ANALYZERS_PREFIX` CI/CD variable. + +For example, the following instructs SAST to pull `my-docker-registry/gitlab-images/bandit` instead +of `registry.gitlab.com/security-products/bandit`: ```yaml include: - template: Security/SAST.gitlab-ci.yml variables: - SECURE_ANALYZERS_PREFIX: my-docker-registry/gl-images + SECURE_ANALYZERS_PREFIX: my-docker-registry/gitlab-images ``` -This configuration requires that your custom registry provides images for all -the official analyzers. +### Disable all default analyzers -### Disabling all default analyzers +You can disable all default SAST analyzers, leaving only [custom analyzers](#custom-analyzers) +enabled. -Setting `SAST_DISABLED` to `true` disables all the official -default analyzers. In `.gitlab-ci.yml` define: +To disable all default analyzers, set the CI/CD variable `SAST_DISABLED` to `true` in your +`.gitlab-ci.yml` file. + +Example: ```yaml include: @@ -92,13 +197,15 @@ variables: SAST_DISABLED: true ``` -That's needed when one totally relies on [custom analyzers](#custom-analyzers). +### Disable specific default analyzers + +Analyzers are run automatically according to the +source code languages detected. However, you can disable select analyzers. -### Disabling specific default analyzers +To disable select analyzers, set the CI/CD variable `SAST_EXCLUDED_ANALYZERS` to a comma-delimited +string listing the analyzers that you want to prevent running. -Set `SAST_EXCLUDED_ANALYZERS` to a comma-delimited string that includes the official -default analyzers that you want to avoid running. In `.gitlab-ci.yml` define the -following to prevent the `eslint` analyzer from running: +For example, to disable the `eslint` analyzer: ```yaml include: @@ -108,27 +215,21 @@ variables: SAST_EXCLUDED_ANALYZERS: "eslint" ``` -## Post Analyzers **(ULTIMATE)** +### Custom analyzers -While analyzers are thin wrappers for executing scanners, post analyzers work to -enrich the data generated within our reports. +You can provide your own analyzers by defining jobs in your CI/CD configuration. For +consistency with the default analyzers, you should add the suffix `-sast` to your custom +SAST jobs. -GitLab SAST post analyzers never modify report contents directly but work by -augmenting results with additional properties (such as CWEs), location tracking fields, -and a means of identifying false positives or insignificant findings. +For more details on integrating a custom security scanner into GitLab, see [Security Scanner Integration](../../../development/integrations/secure.md). -The implementation of post analyzers is determined by feature availability tiers, where -simple data enrichment may occur within our free tier and most advanced processing is split -into separate binaries or pipeline jobs. +#### Example custom analyzer -## Custom Analyzers +This example shows how to add a scanning job that's based on the Docker image +`my-docker-registry/analyzers/csharp`. It runs the script `/analyzer run` and outputs a SAST report +`gl-sast-report.json`. -You can provide your own analyzers by -defining CI jobs in your CI configuration. For consistency, you should suffix your custom -SAST jobs with `-sast`. Here's how to add a scanning job that's based on the -Docker image `my-docker-registry/analyzers/csharp` and generates a SAST report -`gl-sast-report.json` when `/analyzer run` is executed. Define the following in -`.gitlab-ci.yml`: +Define the following in your `.gitlab-ci.yml` file: ```yaml csharp-sast: @@ -140,33 +241,3 @@ csharp-sast: reports: sast: gl-sast-report.json ``` - -The [Security Scanner Integration](../../../development/integrations/secure.md) documentation explains how to integrate custom security scanners into GitLab. - -## Analyzers Data - -| Property / Tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Semgrep | Sobelow | -|--------------------------------|------|--------|----------|-----------------|----------|------------|-------|-----------------|-------|------------|-----------------------|---------------------------|---------|---------| -| Affected item (for example, class or package) | ✓ | ✗ | ✓ | ✗ | ✓ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| Confidence | ✗ | ✓ | ✓ | ✗ | ✓ | x | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✓ | -| Description | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | -| End column | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| End line | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| External ID (for example, CVE) | ✗ | ✗ | ⚠ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | -| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Internal doc/explanation | ✓ | ⚠ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | -| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | -| Severity | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ⚠ | ✗ | -| Solution | ✓ | ✗ | ✗ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | -| Source code extract | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| Start column | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | -| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| URLs | ✓ | ✗ | ✓ | ✗ | ⚠ | ✗ | ⚠ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | - -- ✓ => we have that data -- ⚠ => we have that data but it's partially reliable, or we need to extract it from unstructured content -- ✗ => we don't have that data or it would need to develop specific or inefficient/unreliable logic to obtain it. - -The values provided by these tools are heterogeneous so they are sometimes -normalized into common values (for example, `severity`, `confidence`, and so on). diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 8f006f258b6..38f26b7578d 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -13,12 +13,17 @@ The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab. explains how 4 of the top 6 attacks were application based. Download it to learn how to protect your organization. -If you're using [GitLab CI/CD](../../../ci/index.md), you can use Static Application Security -Testing (SAST) to check your source code for known vulnerabilities. -If the pipeline is associated with a merge request, the SAST analysis is compared with the results of -the target branch's analysis (if available). The results of that comparison are shown in the merge -request. If the pipeline is running from the default branch, the results of the SAST -analysis are available in the [security dashboards](../security_dashboard/index.md). +If you’re using [GitLab CI/CD](../../../ci/index.md), you can use Static Application Security +Testing (SAST) to check your source code for known vulnerabilities. You can run SAST analyzers in +any GitLab tier. The analyzers output JSON-formatted reports as job artifacts. + +With GitLab Ultimate, SAST results are also processed so you can: + +- See them in merge requests. +- Use them in approval workflows. +- Review them in the security dashboard. + +For more details, see the [Summary of features per tier](#summary-of-features-per-tier).  @@ -543,7 +548,7 @@ Several passthrouh types generate a configuration for the target analyzer: - Two `git` passthrough sections pull the head of branch `refs/remotes/origin/test` from the `myrules` Git repository, and revision - `97f7686` from the `sast-rules` Git repostory. From the `sast-rules` Git + `97f7686` from the `sast-rules` Git repository. From the `sast-rules` Git repository, only data from the `go` subdirectory is considered. - The `sast-rules` entry has a higher precedence because it appears later in the configuration. @@ -887,7 +892,7 @@ Some analyzers can be customized with CI/CD variables. | `GRADLE_PATH` | SpotBugs | Path to the `gradle` executable. | | `JAVA_OPTS` | SpotBugs | Additional arguments for the `java` executable. | | `JAVA_PATH` | SpotBugs | Path to the `java` executable. | -| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. Supported versions are `8` and `11`. Defaults to `8`. | +| `SAST_JAVA_VERSION` | SpotBugs | Which Java version to use. [Starting in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/352549), supported versions are `11` and `17` (default). Before GitLab 15.0, supported versions are `8` (default) and `11`. | | `MAVEN_CLI_OPTS` | SpotBugs | Additional arguments for the `mvn` or `mvnw` executable. | | `MAVEN_PATH` | SpotBugs | Path to the `mvn` executable. | | `MAVEN_REPO_PATH` | SpotBugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | @@ -977,19 +982,19 @@ import the following default SAST analyzer images from `registry.gitlab.com` int [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/security-products/sast/bandit:2 -registry.gitlab.com/security-products/sast/brakeman:2 -registry.gitlab.com/security-products/sast/eslint:2 -registry.gitlab.com/security-products/sast/flawfinder:2 -registry.gitlab.com/security-products/sast/gosec:3 -registry.gitlab.com/security-products/sast/kubesec:2 -registry.gitlab.com/security-products/sast/nodejs-scan:2 -registry.gitlab.com/security-products/sast/phpcs-security-audit:2 -registry.gitlab.com/security-products/sast/pmd-apex:2 -registry.gitlab.com/security-products/sast/security-code-scan:2 -registry.gitlab.com/security-products/sast/semgrep:2 -registry.gitlab.com/security-products/sast/sobelow:2 -registry.gitlab.com/security-products/sast/spotbugs:2 +registry.gitlab.com/security-products/bandit:2 +registry.gitlab.com/security-products/brakeman:2 +registry.gitlab.com/security-products/eslint:2 +registry.gitlab.com/security-products/flawfinder:2 +registry.gitlab.com/security-products/gosec:3 +registry.gitlab.com/security-products/kubesec:2 +registry.gitlab.com/security-products/nodejs-scan:2 +registry.gitlab.com/security-products/phpcs-security-audit:2 +registry.gitlab.com/security-products/pmd-apex:2 +registry.gitlab.com/security-products/security-code-scan:2 +registry.gitlab.com/security-products/semgrep:2 +registry.gitlab.com/security-products/sobelow:2 +registry.gitlab.com/security-products/spotbugs:2 ``` The process for importing Docker images into a local offline Docker registry depends on diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 0a18e7d5f45..3937cbd77b6 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -195,13 +195,18 @@ Secret Detection can be customized by defining available CI/CD variables: | CI/CD variable | Default value | Description | |-----------------------------------|---------------|-------------| -| `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | -| `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | -| `SECRET_DETECTION_COMMITS` | - | The list of commits that Gitleaks should scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. | | `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | | `SECRET_DETECTION_IMAGE_SUFFIX` | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. | +In previous GitLab versions, the following variables were also available: + +| CI/CD variable | Default value | Description | +|-----------------------------------|---------------|-------------| +| `SECRET_DETECTION_COMMIT_FROM` | - | The commit a Gitleaks scan starts at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | +| `SECRET_DETECTION_COMMIT_TO` | - | The commit a Gitleaks scan ends at. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. Replaced with `SECRET_DETECTION_COMMITS`. | +| `SECRET_DETECTION_COMMITS` | - | The list of commits that Gitleaks should scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/243564) in GitLab 13.5. [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/352565) in GitLab 15.0. | + ### Custom rulesets **(ULTIMATE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211387) in GitLab 13.5. diff --git a/doc/user/application_security/secret_detection/post_processing.md b/doc/user/application_security/secret_detection/post_processing.md index 643da47d876..9771658da4e 100644 --- a/doc/user/application_security/secret_detection/post_processing.md +++ b/doc/user/application_security/secret_detection/post_processing.md @@ -46,7 +46,7 @@ sequenceDiagram Cloud Vendor-->>+RevocationAPI: ACCEPTED ``` -## Integrate your cloud provider service with GitLab Saas +## Integrate your cloud provider service with GitLab SaaS Third party cloud and SaaS providers can [express integration interest by filling out this form](https://forms.gle/wWpvrtLRK21Q2WJL9). diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 488ec336646..577606885ca 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -17,6 +17,7 @@ To use the Security Dashboards, you must: - Configure jobs to use the [`reports` syntax](../../../ci/yaml/index.md#artifactsreports). - Use [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or later. If you use the shared runners on GitLab.com, you are using the correct version. +- Have the [correct role](../../permissions.md) for the project or group. ## When Security Dashboards are updated diff --git a/doc/user/application_security/threat_monitoring/img/threat_monitoring_policy_alert_list_v14_3.png b/doc/user/application_security/threat_monitoring/img/threat_monitoring_policy_alert_list_v14_3.png Binary files differdeleted file mode 100644 index a11a7fafc4a..00000000000 --- a/doc/user/application_security/threat_monitoring/img/threat_monitoring_policy_alert_list_v14_3.png +++ /dev/null diff --git a/doc/user/application_security/threat_monitoring/index.md b/doc/user/application_security/threat_monitoring/index.md deleted file mode 100644 index 9b8dd2825ea..00000000000 --- a/doc/user/application_security/threat_monitoring/index.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -type: reference, howto -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Threat Monitoring **(ULTIMATE)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14707) in GitLab 12.9. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Threat Monitoring is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The **Threat Monitoring** page provides alerts and metrics -for the GitLab application runtime security features. You can access -these by navigating to your project's **Security & Compliance > Threat -Monitoring** page. - -GitLab supports statistics for the following security features: - -- [Container Network Policies](../../../topics/autodevops/stages.md#network-policy) - -## Container Network Policy Alert list - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3438) in GitLab 13.9. - -The policy alert list displays your policy's alert activity. You can sort the list by these columns: - -- Date and time -- Events -- Status - -You can filter the list with the **Policy Name** filter and the **Status** filter at the top. Use -the selector menu in the **Status** column to set the status for each alert: - -- Unreviewed -- In review -- Resolved -- Dismissed - -By default, the list doesn't display resolved or dismissed alerts. - - - -Clicking an alert's row opens the alert drawer, which shows more information about the alert. A user -can also create an incident from the alert and update the alert status in the alert drawer. - -Clicking an alert's name takes the user to the [alert details page](../../../operations/incident_management/alerts.md#alert-details-page). diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index 18b99e06299..87344e4ff65 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -42,7 +42,7 @@ A vulnerability's status can be one of the following: | Dismissed | A user has seen this vulnerability and dismissed it because it is not accurate or otherwise not to be resolved. | | Resolved | The vulnerability has been fixed or is no longer present. | -Dismissed vulnerabilities are ignored if detected in subsequent scans. Resolved vulnerabilities that are reintroduced and detected by subsequent scans have a _new_ vulnerability record created. When an existing vulnerability is no longer detected in a project's `default` branch, you should change its status to Resolved. This ensures that if it is accidentally reintroduced in a future merge, it will be visible again as a new record. You can use the [Activity filter](../vulnerability_report/#activity-filter) to select all vulnerabilities that are no longer detected, and [change their status](../vulnerability_report#change-status-of-multiple-vulnerabilities). +Dismissed vulnerabilities are ignored if detected in subsequent scans. Resolved vulnerabilities that are reintroduced and detected by subsequent scans have a _new_ vulnerability record created. When an existing vulnerability is no longer detected in a project's `default` branch, you should change its status to Resolved. This ensures that if it is accidentally reintroduced in a future merge, it will be visible again as a new record. You can use the [Activity filter](../vulnerability_report/#activity-filter) to select all vulnerabilities that are no longer detected, and [change their status](../vulnerability_report#change-status-of-vulnerabilities). ## Change vulnerability status diff --git a/doc/user/application_security/vulnerabilities/severities.md b/doc/user/application_security/vulnerabilities/severities.md index 89464064ea3..967a6d9fa89 100644 --- a/doc/user/application_security/vulnerabilities/severities.md +++ b/doc/user/application_security/vulnerabilities/severities.md @@ -56,8 +56,6 @@ the following tables: | GitLab analyzer | Outputs severity levels? | Native severity level type | Native severity level example | |------------------------------------------------------------------------------------------|------------------------------|----------------------------|-------------------------------------| -| [`bundler-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/bundler-audit) | **{check-circle}** Yes | String | `low`, `medium`, `high`, `critical` | -| [`retire.js`](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) | **{check-circle}** Yes | String | `low`, `medium`, `high`, `critical` | | [`gemnasium`](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) | **{check-circle}** Yes | CVSS v2.0 Rating and CVSS v3.1 Qualitative Severity Rating | `(AV:N/AC:L/Au:S/C:P/I:P/A:N)`, `CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:C/C:H/I:H/A:H` | ## Container Scanning diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index a9cef15e3e8..e499ddbbd6b 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -11,7 +11,7 @@ The Vulnerability Report provides information about vulnerabilities from scans o The scan results from a pipeline are only ingested after all the jobs in the pipeline complete. Partial results for a pipeline with jobs in progress can be seen in the pipeline security tab. -The report is available for projects, groups, and the Security Center. +The report is available for users with the [correct role](../../permissions.md) on projects, groups, and the Security Center. At all levels, the Vulnerability Report contains: @@ -34,13 +34,18 @@ in that row: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/6165) in GitLab 11.1. -The project-level Vulnerability Report also contains: +At the project level, the Vulnerability Report also contains: - A time stamp showing when it was updated, including a link to the latest pipeline. - The number of failures that occurred in the most recent pipeline. Select the failure notification to view the **Failed jobs** tab of the pipeline's page. -To access the report, navigate to **Security & Compliance > Vulnerability Report**. +### View the project-level vulnerability report + +To view the project-level vulnerability report: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. ## Vulnerability Report actions @@ -56,17 +61,22 @@ From the Vulnerability Report you can: ## Vulnerability Report filters -You can filter the vulnerabilities table by: +You can filter the Vulnerability Report to narrow focus on only vulnerabilities matching specific +criteria. + +The available filters are: <!-- vale gitlab.SubstitutionWarning = NO --> -| Filter | Available options | -|:---------|:------------------| -| Status | Detected, Confirmed, Dismissed, Resolved. | -| Severity | Critical, High, Medium, Low, Info, Unknown. | -| Tool | For more details, see [Tool filter](#tool-filter). | -| Project | For more details, see [Project filter](#project-filter). | -| Activity | For more details, see [Activity filter](#activity-filter). | +- **Status**: Detected, Confirmed, Dismissed, Resolved. +- **Severity**: Critical, High, Medium, Low, Info, Unknown. +- **Tool**: For more details, see [Tool filter](#tool-filter). +- **Project**: For more details, see [Project filter](#project-filter). +- **Activity**: For more details, see [Activity filter](#activity-filter). + +The filters' criteria are combined to show only vulnerabilities matching all criteria. +An exception to this behavior is the Activity filter. For more details about how it works, see +[Activity filter](#activity-filter). <!-- vale gitlab.SubstitutionWarning = YES --> @@ -75,7 +85,7 @@ You can filter the vulnerabilities table by: To filter the list of vulnerabilities: 1. Select a filter. -1. Select values from the dropdown. +1. Select values from the dropdown list. 1. Repeat the above steps for each desired filter. After each filter is selected: @@ -83,11 +93,9 @@ After each filter is selected: - The list of matching vulnerabilities is updated. - The vulnerability severity totals are updated. -The filters' criteria are combined to show only vulnerabilities matching all criteria. -An exception to this behavior is the Activity filter. For more details about how it works, see -[Activity filter](#activity-filter). +### Tool filter -## Tool filter +> The third-party tool filter was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229661) in GitLab 13.12. The tool filter allows you to focus on vulnerabilities detected by selected tools. @@ -95,7 +103,7 @@ When using the tool filter, you can choose: - **All tools** (default). - Individual GitLab-provided tools. -- Any integrated 3rd-party tool. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229661) in GitLab 13.12. +- Any integrated third-party tool. For details of each of the available tools, see [Security scanning tools](../index.md#security-scanning-tools). @@ -103,11 +111,9 @@ For details of each of the available tools, see [Security scanning tools](../ind The content of the Project filter depends on the current level: -| Level | Content of the Project filter | -|:---------------|:------------------------------| -| Security Center | Only projects you've [added to your personal Security Center](../security_dashboard/index.md#add-projects-to-the-security-center). | -| Group level | All projects in the group. | -| Project level | Not applicable. | +- **Security Center**: Only projects you've [added to your personal Security Center](../security_dashboard/index.md#add-projects-to-the-security-center). +- **Group level**: All projects in the group. +- **Project level**: Not applicable. ### Activity filter @@ -119,13 +125,11 @@ all options can be selected in combination. Selection behavior when using the Activity filter: -| Activity selection | Results displayed | -|:------------------------------------|:------------------| -| All | Vulnerabilities with any Activity status (same as ignoring this filter). Selecting this deselects any other Activity filter options. | -| No activity | Only vulnerabilities without either an associated Issue or that are no longer detected. Selecting this deselects any other Activity filter options. | -| With issues | Only vulnerabilities with one or more associated issues. Does not include vulnerabilities that also are no longer detected. | -| No longer detected | Only vulnerabilities that are no longer detected in the latest pipeline scan of the `default` branch. Does not include vulnerabilities with one or more associated issues. | -| With issues and No longer detected | Only vulnerabilities that have one or more associated issues and also are no longer detected in the latest pipeline scan of the `default` branch. | +- **All**: Vulnerabilities with any Activity status (same as ignoring this filter). Selecting this deselects any other Activity filter options. +- **No activity**: Only vulnerabilities without either an associated issue or that are no longer detected. Selecting this deselects any other Activity filter options. +- **With issues**: Only vulnerabilities with one or more associated issues. Does not include vulnerabilities that also are no longer detected. +- **No longer detected**: Only vulnerabilities that are no longer detected in the latest pipeline scan of the `default` branch. Does not include vulnerabilities with one or more associated issues. +- **With issues** and **No longer detected**: Only vulnerabilities that have one or more associated issues and also are no longer detected in the latest pipeline scan of the `default` branch. ## View details of a vulnerability @@ -155,23 +159,32 @@ If Jira issue support is enabled, the issue link found in the Activity entry lin > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292636) in GitLab 13.10, all statuses became selectable. +From the Vulnerability Report you can change the status of one or more vulnerabilities. + To change the status of vulnerabilities in the table: -1. Select the checkbox for each vulnerability you want to update the status of. -1. In the dropdown that appears select the desired status, then select **Change status**. +1. Select the checkbox beside each vulnerability you want to update the status of. To select all, + select the checkbox in the table header. +1. In the **Set status** dropdown list, select the desired status. +1. Select **Change status**.  -### Change status of multiple vulnerabilities +## Dismissing a vulnerability -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35816) in GitLab 12.9. +When you evaluate a vulnerability and decide it requires no more action, you can mark it +as **Dismissed**. Dismissed vulnerabilities don't appear in the merge request security widget +when detected in future scans. -You can change the status of multiple vulnerabilities at once: +When a vulnerability is dismissed, a record is made of: -1. In the list of vulnerabilities, select the checkbox for each vulnerability you want to update. - To select all, select the checkbox in the table header. -1. Above the table, select a new status. -1. Click **Change status** to save. +- Who dismissed it. +- Date and time when it was dismissed. +- Optionally, a reason why it was dismissed. + +Vulnerability records cannot be deleted, so a permanent record always remains. + +If a vulnerability is dismissed in error, reverse the dismissal by changing its status. ## Export vulnerability details @@ -190,13 +203,19 @@ Fields included are: - Scanner name - Status - Vulnerability -- Details +- Basic details - Additional information - Severity - [CVE](https://cve.mitre.org/) (Common Vulnerabilities and Exposures) - [CWE](https://cwe.mitre.org/) (Common Weakness Enumeration) - Other identifiers +NOTE: +Full details are available through our +[Job Artifacts API](../../../api/job_artifacts.md#download-a-single-artifact-file-from-specific-tag-or-branch). +Use one of the `gl-*-report.json` report filenames in place of `*artifact_path` +to obtain, for example, the path of files in which vulnerabilities were detected. + ### Export details in CSV format To export details of all vulnerabilities listed in the Vulnerability Report, select **Export**. @@ -224,6 +243,7 @@ To undo this action, select a different status from the same menu. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301003) in GitLab 14.9. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/353796) in GitLab 14.10. +> - [Feature flag `new_vulnerability_form`](https://gitlab.com/gitlab-org/gitlab/-/issues/359049) removed in GitLab 15.0. To add a new vulnerability finding from your project level Vulnerability Report page: diff --git a/doc/user/clusters/agent/ci_cd_tunnel.md b/doc/user/clusters/agent/ci_cd_tunnel.md index c15041f6b0d..1b99fcf9739 100644 --- a/doc/user/clusters/agent/ci_cd_tunnel.md +++ b/doc/user/clusters/agent/ci_cd_tunnel.md @@ -1,261 +1,11 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'ci_cd_workflow.md' +remove_date: '2022-07-20' --- -# Using a GitLab CI/CD workflow for Kubernetes **(FREE)** +This document was moved to [another location](ci_cd_workflow.md). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327409) in GitLab 14.1. -> - The pre-configured `KUBECONFIG` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324275) in GitLab 14.2. -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) the `ci_access` attribute in GitLab 14.3. -> - The ability to authorize groups was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. -> - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) to GitLab Free in 14.5. -> - Support for Omnibus installations was [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5686) in GitLab 14.5. -> - The ability to switch between certificate-based clusters and agents was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335089) in GitLab 14.9. The certificate-based cluster context is always called `gitlab-deploy`. - -You can use a GitLab CI/CD workflow to safely deploy to and update your Kubernetes clusters. - -To do so, you must first [install an agent in your cluster](install/index.md). When done, you have a Kubernetes context and can -run Kubernetes API commands in your GitLab CI/CD pipeline. - -To ensure access to your cluster is safe: - -- Each agent has a separate context (`kubecontext`). -- Only the project where the agent is configured, and any additional projects you authorize, can access the agent in your cluster. - -You do not need to have a runner in the cluster with the agent. - -## GitLab CI/CD workflow steps - -To update a Kubernetes cluster by using GitLab CI/CD, complete the following steps. - -1. Ensure you have a working Kubernetes cluster and the manifests are in a GitLab project. -1. In the same GitLab project, [register and install the GitLab agent](install/index.md). -1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to - select the agent's Kubernetes context and run the Kubernetes API commands. -1. Run your pipeline to deploy to or update the cluster. - -If you have multiple GitLab projects that contain Kubernetes manifests: - -1. [Install the GitLab agent](install/index.md) in its own project, or in one of the - GitLab projects where you keep Kubernetes manifests. -1. [Authorize the agent](#authorize-the-agent) to access your GitLab projects. -1. Optional. For added security, [use impersonation](#use-impersonation-to-restrict-project-and-group-access). -1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to - select the agent's Kubernetes context and run the Kubernetes API commands. -1. Run your pipeline to deploy to or update the cluster. - -## Authorize the agent - -You must authorize the agent to access the project where you keep your Kubernetes manifests. -You can authorize the agent to access individual projects, or authorize a group or subgroup, -so all projects within have access. For added security, you can also -[use impersonation](#use-impersonation-to-restrict-project-and-group-access). - -### Authorize the agent to access your projects - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327850) in GitLab 14.4. - -To authorize the agent to access the GitLab project where you keep Kubernetes manifests: - -1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). -1. Edit the file. Under the `ci_access` keyword, add the `projects` attribute. -1. For the `id`, add the path: - - ```yaml - ci_access: - projects: - - id: path/to/project - ``` - - - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. - - You can install additional agents into the same cluster to accommodate additional hierarchies. - - You can authorize up to 100 projects. - -All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. -Choose the context to run `kubectl` commands from your CI/CD scripts. - -### Authorize the agent to access projects in your groups - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. - -To authorize the agent to access all of the GitLab projects in a group or subgroup: - -1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). -1. Edit the file. Under the `ci_access` keyword, add the `groups` attribute. -1. For the `id`, add the path: - - ```yaml - ci_access: - groups: - - id: path/to/group/subgroup - ``` - - - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. - - You can install additional agents into the same cluster to accommodate additional hierarchies. - - All of the subgroups of an authorized group also have access to the same agent (without being specified individually). - - You can authorize up to 100 groups. - -All the projects that belong to the group and its subgroups are now authorized to access the agent. -All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. -Choose the context to run `kubectl` commands from your CI/CD scripts. - -## Update your `.gitlab-ci.yml` file to run `kubectl` commands - -In the project where you want to run Kubernetes commands, edit your project's `.gitlab-ci.yml` file. - -In the first command under the `script` keyword, set your agent's context. -Use the format `path/to/agent/repository:agent-name`. For example: - -```yaml - deploy: - image: - name: bitnami/kubectl:latest - entrypoint: [""] - script: - - kubectl config get-contexts - - kubectl config use-context path/to/agent/repository:agent-name - - kubectl get pods -``` - -If you are not sure what your agent's context is, open a terminal and connect to your cluster. -Run `kubectl config get-contexts`. - -### Environments with both certificate-based and agent-based connections - -When you deploy to an environment that has both a [certificate-based -cluster](../../infrastructure/clusters/index.md) (deprecated) and an agent connection: - -- The certificate-based cluster's context is called `gitlab-deploy`. This context - is always selected by default. -- In GitLab 14.9 and later, agent contexts are included in the - `KUBECONFIG`. You can select them by using `kubectl config use-context - path/to/agent/repository:agent-name`. -- In GitLab 14.8 and earlier, you can still use agent connections, but for environments that - already have a certificate-based cluster, the agent connections are not included in the `KUBECONFIG`. - -To use an agent connection when certificate-based connections are present, you can manually configure a new `kubectl` -configuration context. For example: - - ```yaml - deploy: - variables: - KUBE_CONTEXT: my-context # The name to use for the new context - AGENT_ID: 1234 # replace with your agent's numeric ID - K8S_PROXY_URL: wss://kas.gitlab.com/k8s-proxy/ # replace with your agent server (KAS) Kubernetes proxy URL - # ... any other variables you have configured - before_script: - - kubectl config set-credentials agent:$AGENT_ID --token="ci:${AGENT_ID}:${CI_JOB_TOKEN}" - - kubectl config set-cluster gitlab --server="${K8S_PROXY_URL}" - - kubectl config set-context "$KUBE_CONTEXT" --cluster=gitlab --user="agent:${AGENT_ID}" - - kubectl config use-context "$KUBE_CONTEXT" - # ... rest of your job configuration - ``` - -## Use impersonation to restrict project and group access **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345014) in GitLab 14.5. - -By default, your CI/CD job inherits all the permissions from the service account used to install the -agent in the cluster. -To restrict access to your cluster, you can use [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). - -To specify impersonations, use the `access_as` attribute in your agent configuration file and use Kubernetes RBAC rules to manage impersonated account permissions. - -You can impersonate: - -- The agent itself (default). -- The CI/CD job that accesses the cluster. -- A specific user or system account defined within the cluster. - -### Impersonate the agent - -The agent is impersonated by default. You don't need to do anything to impersonate it. - -### Impersonate the CI/CD job that accesses the cluster - -To impersonate the CI/CD job that accesses the cluster, under the `access_as` key, add the `ci_job: {}` key-value. - -When the agent makes the request to the actual Kubernetes API, it sets the -impersonation credentials in the following way: - -- `UserName` is set to `gitlab:ci_job:<job id>`. Example: `gitlab:ci_job:1074499489`. -- `Groups` is set to: - - `gitlab:ci_job` to identify all requests coming from CI jobs. - - The list of IDs of groups the project is in. - - The project ID. - - The slug of the environment this job belongs to. - - Example: for a CI job in `group1/group1-1/project1` where: - - - Group `group1` has ID 23. - - Group `group1/group1-1` has ID 25. - - Project `group1/group1-1/project1` has ID 150. - - Job running in a prod environment. - - Group list would be `[gitlab:ci_job, gitlab:group:23, gitlab:group:25, gitlab:project:150, gitlab:project_env:150:prod]`. - -- `Extra` carries extra information about the request. The following properties are set on the impersonated identity: - -| Property | Description | -| -------- | ----------- | -| `agent.gitlab.com/id` | Contains the agent ID. | -| `agent.gitlab.com/config_project_id` | Contains the agent's configuration project ID. | -| `agent.gitlab.com/project_id` | Contains the CI project ID. | -| `agent.gitlab.com/ci_pipeline_id` | Contains the CI pipeline ID. | -| `agent.gitlab.com/ci_job_id` | Contains the CI job ID. | -| `agent.gitlab.com/username` | Contains the username of the user the CI job is running as. | -| `agent.gitlab.com/environment_slug` | Contains the slug of the environment. Only set if running in an environment. | - -Example to restrict access by the CI/CD job's identity: - -```yaml -ci_access: - projects: - - id: path/to/project - access_as: - ci_job: {} -``` - -### Impersonate a static identity - -For a given connection, you can use a static identity for the impersonation. - -Under the `access_as` key, add the `impersonate` key to make the request using the provided identity. - -The identity can be specified with the following keys: - -- `username` (required) -- `uid` -- `groups` -- `extra` - -See the [official Kubernetes documentation for details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). - -## Troubleshooting - -### `kubectl` commands not supported - -The commands `kubectl exec`, `kubectl cp`, and `kubectl attach` are not supported. -Anything that uses these API endpoints does not work, because they use the deprecated -SPDY protocol. -[An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/346248) to add support for these commands. - -### Grant write permissions to `~/.kube/cache` - -Tools like `kubectl`, Helm, `kpt`, and `kustomize` cache information about -the cluster in `~/.kube/cache`. If this directory is not writable, the tool fetches information on each invocation, -making interactions slower and creating unnecessary load on the cluster. For the best experience, in the -image you use in your .`gitlab-ci.yml` file, ensure this directory is writable. - -### Enable TLS - -If you are on a self-managed GitLab instance, ensure your instance is configured with Transport Layer Security (TLS). - -If you attempt to use `kubectl` without TLS, you might get an error like: - -```shell -$ kubectl get pods -error: You must be logged in to the server (the server has asked for the client to provide credentials) -``` +<!-- This redirect file can be deleted after <2022-07-20>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/user/clusters/agent/ci_cd_workflow.md b/doc/user/clusters/agent/ci_cd_workflow.md new file mode 100644 index 00000000000..644a753e282 --- /dev/null +++ b/doc/user/clusters/agent/ci_cd_workflow.md @@ -0,0 +1,263 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Using GitLab CI/CD with a Kubernetes cluster **(FREE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327409) in GitLab 14.1. +> - The pre-configured `KUBECONFIG` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324275) in GitLab 14.2. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) the `ci_access` attribute in GitLab 14.3. +> - The ability to authorize groups was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. +> - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) to GitLab Free in 14.5. +> - Support for Omnibus installations was [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5686) in GitLab 14.5. +> - The ability to switch between certificate-based clusters and agents was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335089) in GitLab 14.9. The certificate-based cluster context is always called `gitlab-deploy`. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80508) from _CI/CD tunnel_ to _CI/CD workflow_ in GitLab 14.9. + +You can use a GitLab CI/CD workflow to safely deploy to and update your Kubernetes clusters. + +To do so, you must first [install an agent in your cluster](install/index.md). When done, you have a Kubernetes context and can +run Kubernetes API commands in your GitLab CI/CD pipeline. + +To ensure access to your cluster is safe: + +- Each agent has a separate context (`kubecontext`). +- Only the project where the agent is configured, and any additional projects you authorize, can access the agent in your cluster. + +You do not need to have a runner in the cluster with the agent. + +## GitLab CI/CD workflow steps + +To update a Kubernetes cluster by using GitLab CI/CD, complete the following steps. + +1. Ensure you have a working Kubernetes cluster and the manifests are in a GitLab project. +1. In the same GitLab project, [register and install the GitLab agent](install/index.md). +1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to + select the agent's Kubernetes context and run the Kubernetes API commands. +1. Run your pipeline to deploy to or update the cluster. + +If you have multiple GitLab projects that contain Kubernetes manifests: + +1. [Install the GitLab agent](install/index.md) in its own project, or in one of the + GitLab projects where you keep Kubernetes manifests. +1. [Authorize the agent](#authorize-the-agent) to access your GitLab projects. +1. Optional. For added security, [use impersonation](#use-impersonation-to-restrict-project-and-group-access). +1. [Update your `.gitlab-ci.yml` file](#update-your-gitlab-ciyml-file-to-run-kubectl-commands) to + select the agent's Kubernetes context and run the Kubernetes API commands. +1. Run your pipeline to deploy to or update the cluster. + +## Authorize the agent + +You must authorize the agent to access the project where you keep your Kubernetes manifests. +You can authorize the agent to access individual projects, or authorize a group or subgroup, +so all projects within have access. For added security, you can also +[use impersonation](#use-impersonation-to-restrict-project-and-group-access). + +### Authorize the agent to access your projects + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327850) in GitLab 14.4. + +To authorize the agent to access the GitLab project where you keep Kubernetes manifests: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). +1. Edit the `config.yaml` file. Under the `ci_access` keyword, add the `projects` attribute. +1. For the `id`, add the path: + + ```yaml + ci_access: + projects: + - id: path/to/project + ``` + + - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. + - You can install additional agents into the same cluster to accommodate additional hierarchies. + - You can authorize up to 100 projects. + +All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. +Choose the context to run `kubectl` commands from your CI/CD scripts. + +### Authorize the agent to access projects in your groups + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. + +To authorize the agent to access all of the GitLab projects in a group or subgroup: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file (`config.yaml`). +1. Edit the `config.yaml` file. Under the `ci_access` keyword, add the `groups` attribute. +1. For the `id`, add the path: + + ```yaml + ci_access: + groups: + - id: path/to/group/subgroup + ``` + + - The Kubernetes projects must be in the same group hierarchy as the project where the agent's configuration is. + - You can install additional agents into the same cluster to accommodate additional hierarchies. + - All of the subgroups of an authorized group also have access to the same agent (without being specified individually). + - You can authorize up to 100 groups. + +All the projects that belong to the group and its subgroups are now authorized to access the agent. +All CI/CD jobs now include a `KUBECONFIG` with contexts for every shared agent connection. +Choose the context to run `kubectl` commands from your CI/CD scripts. + +## Update your `.gitlab-ci.yml` file to run `kubectl` commands + +In the project where you want to run Kubernetes commands, edit your project's `.gitlab-ci.yml` file. + +In the first command under the `script` keyword, set your agent's context. +Use the format `path/to/agent/repository:agent-name`. For example: + +```yaml +deploy: + image: + name: bitnami/kubectl:latest + entrypoint: [''] + script: + - kubectl config get-contexts + - kubectl config use-context path/to/agent/repository:agent-name + - kubectl get pods +``` + +If you are not sure what your agent's context is, open a terminal and connect to your cluster. +Run `kubectl config get-contexts`. + +### Environments with both certificate-based and agent-based connections + +When you deploy to an environment that has both a [certificate-based +cluster](../../infrastructure/clusters/index.md) (deprecated) and an agent connection: + +- The certificate-based cluster's context is called `gitlab-deploy`. This context + is always selected by default. +- In GitLab 14.9 and later, agent contexts are included in the + `KUBECONFIG`. You can select them by using `kubectl config use-context path/to/agent/repository:agent-name`. +- In GitLab 14.8 and earlier, you can still use agent connections, but for environments that + already have a certificate-based cluster, the agent connections are not included in the `KUBECONFIG`. + +To use an agent connection when certificate-based connections are present, you can manually configure a new `kubectl` +configuration context. For example: + +```yaml +deploy: + variables: + KUBE_CONTEXT: my-context # The name to use for the new context + AGENT_ID: 1234 # replace with your agent's numeric ID + K8S_PROXY_URL: https://<KAS_DOMAIN>/k8s-proxy/ # For agent server (KAS) deployed in Kubernetes cluster (for gitlab.com use kas.gitlab.com); replace with your URL + # K8S_PROXY_URL: https://<GITLAB_DOMAIN>/-/kubernetes-agent/k8s-proxy/ # For agent server (KAS) in Omnibus + # ... any other variables you have configured + before_script: + - kubectl config set-credentials agent:$AGENT_ID --token="ci:${AGENT_ID}:${CI_JOB_TOKEN}" + - kubectl config set-cluster gitlab --server="${K8S_PROXY_URL}" + - kubectl config set-context "$KUBE_CONTEXT" --cluster=gitlab --user="agent:${AGENT_ID}" + - kubectl config use-context "$KUBE_CONTEXT" + # ... rest of your job configuration +``` + +## Use impersonation to restrict project and group access **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345014) in GitLab 14.5. + +By default, your CI/CD job inherits all the permissions from the service account used to install the +agent in the cluster. +To restrict access to your cluster, you can use [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). + +To specify impersonations, use the `access_as` attribute in your agent configuration file and use Kubernetes RBAC rules to manage impersonated account permissions. + +You can impersonate: + +- The agent itself (default). +- The CI/CD job that accesses the cluster. +- A specific user or system account defined within the cluster. + +### Impersonate the agent + +The agent is impersonated by default. You don't need to do anything to impersonate it. + +### Impersonate the CI/CD job that accesses the cluster + +To impersonate the CI/CD job that accesses the cluster, under the `access_as` key, add the `ci_job: {}` key-value. + +When the agent makes the request to the actual Kubernetes API, it sets the +impersonation credentials in the following way: + +- `UserName` is set to `gitlab:ci_job:<job id>`. Example: `gitlab:ci_job:1074499489`. +- `Groups` is set to: + + - `gitlab:ci_job` to identify all requests coming from CI jobs. + - The list of IDs of groups the project is in. + - The project ID. + - The slug of the environment this job belongs to. + + Example: for a CI job in `group1/group1-1/project1` where: + + - Group `group1` has ID 23. + - Group `group1/group1-1` has ID 25. + - Project `group1/group1-1/project1` has ID 150. + - Job running in a prod environment. + + Group list would be `[gitlab:ci_job, gitlab:group:23, gitlab:group:25, gitlab:project:150, gitlab:project_env:150:prod]`. + +- `Extra` carries extra information about the request. The following properties are set on the impersonated identity: + +| Property | Description | +| ------------------------------------ | ---------------------------------------------------------------------------- | +| `agent.gitlab.com/id` | Contains the agent ID. | +| `agent.gitlab.com/config_project_id` | Contains the agent's configuration project ID. | +| `agent.gitlab.com/project_id` | Contains the CI project ID. | +| `agent.gitlab.com/ci_pipeline_id` | Contains the CI pipeline ID. | +| `agent.gitlab.com/ci_job_id` | Contains the CI job ID. | +| `agent.gitlab.com/username` | Contains the username of the user the CI job is running as. | +| `agent.gitlab.com/environment_slug` | Contains the slug of the environment. Only set if running in an environment. | + +Example `config.yaml` to restrict access by the CI/CD job's identity: + +```yaml +ci_access: + projects: + - id: path/to/project + access_as: + ci_job: {} +``` + +### Impersonate a static identity + +For a given connection, you can use a static identity for the impersonation. + +Under the `access_as` key, add the `impersonate` key to make the request using the provided identity. + +The identity can be specified with the following keys: + +- `username` (required) +- `uid` +- `groups` +- `extra` + +See the [official Kubernetes documentation for details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). + +## Troubleshooting + +### `kubectl` commands not supported + +The commands `kubectl exec`, `kubectl cp`, `kubectl attach`, `kubectl run --attach=true` and `kubectl port-forward` are not supported. +Anything that uses these API endpoints does not work, because they use the deprecated +SPDY protocol. +[An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/346248) to add support for these commands. + +### Grant write permissions to `~/.kube/cache` + +Tools like `kubectl`, Helm, `kpt`, and `kustomize` cache information about +the cluster in `~/.kube/cache`. If this directory is not writable, the tool fetches information on each invocation, +making interactions slower and creating unnecessary load on the cluster. For the best experience, in the +image you use in your `.gitlab-ci.yml` file, ensure this directory is writable. + +### Enable TLS + +If you are on a self-managed GitLab instance, ensure your instance is configured with Transport Layer Security (TLS). + +If you attempt to use `kubectl` without TLS, you might get an error like: + +```shell +$ kubectl get pods +error: You must be logged in to the server (the server has asked for the client to provide credentials) +``` diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index e99e3b00ec7..6ca9d855b44 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Using a GitOps workflow for Kubernetes **(PREMIUM)** +# Using GitOps with a Kubernetes cluster **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added. diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index bab3f3137fe..d54d432f0f5 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -35,7 +35,7 @@ In a [**GitOps** workflow](gitops.md), you keep your Kubernetes manifests in Git any time you update your manifests, the agent updates the cluster. This workflow is fully driven with Git and is considered pull-based, because the cluster is pulling updates from your GitLab repository. -In a [**CI/CD** workflow](ci_cd_tunnel.md), you use GitLab CI/CD to query and update your cluster by using the Kubernetes API. +In a [**CI/CD** workflow](ci_cd_workflow.md), you use GitLab CI/CD to query and update your cluster by using the Kubernetes API. This workflow is considered push-based, because GitLab is pushing requests from GitLab CI/CD to your cluster. ## Supported cluster versions @@ -43,6 +43,7 @@ This workflow is considered push-based, because GitLab is pushing requests from GitLab supports the following Kubernetes versions. You can upgrade your Kubernetes version to a supported version at any time: +- 1.22 (support ends on March 22, 2023) - 1.21 (support ends on November 22, 2022) - 1.20 (support ends on July 22, 2022) @@ -65,7 +66,7 @@ Read about how to [migrate to the agent for Kubernetes](../../infrastructure/clu ## Related topics - [GitOps workflow](gitops.md) -- [GitLab CI/CD workflow](ci_cd_tunnel.md) +- [GitLab CI/CD workflow](ci_cd_workflow.md) - [Install the agent](install/index.md) - [Work with the agent](repository.md) - [Troubleshooting](troubleshooting.md) diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index e76ef9e827d..f747c6c0e25 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -20,39 +20,51 @@ Before you can install the agent in your cluster, you need: - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/quickstart) - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) -- On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). +- On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). Then it will be available by default at `wss://gitlab.example.com/-/kubernetes-agent/`. On GitLab.com, the agent server is available at `wss://kas.gitlab.com`. ## Installation steps To install the agent in your cluster: +1. [Choose a name for the agent](#agent-naming-convention). 1. [Register the agent with GitLab](#register-the-agent-with-gitlab). 1. [Install the agent in your cluster](#install-the-agent-in-the-cluster). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a GitLab 14.2 [walk-through of this process](https://www.youtube.com/watch?v=XuBpKtsgGkE). +### Agent naming convention + +The agent name must follow the [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). +The name must: + +- Be unique in the project. +- Contain at most 63 characters. +- Contain only lowercase alphanumeric characters or `-`. +- Start with an alphanumeric character. +- End with an alphanumeric character. + ### Register the agent with GitLab > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5786) in GitLab 14.1, you can create a new agent record directly from the GitLab UI. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347240) in GitLab 14.9, the agent can be registered without creating an agent configuration file. -You must register an agent with GitLab. - FLAG: In GitLab 14.10, a [flag](../../../../administration/feature_flags.md) named `certificate_based_clusters` changed the **Actions** menu to focus on the agent rather than certificates. The flag is [enabled on GitLab.com and self-managed](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). Prerequisites: -- For a [GitLab CI/CD workflow](../ci_cd_tunnel.md), ensure that +- For a [GitLab CI/CD workflow](../ci_cd_workflow.md), ensure that [GitLab CI/CD is enabled](../../../../ci/enable_or_disable_ci.md#enable-cicd-in-a-project). -To register an agent with GitLab: +You must register an agent before you can install the agent in your cluster. To register an agent: 1. On the top bar, select **Menu > Projects** and find your project. + If you have an [agent configuration file](#create-an-agent-configuration-file), + it must be in this project. Your cluster manifest files should also be in this project. 1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. 1. Select **Connect a cluster (agent)**. - - If you want to create a configuration with CI/CD defaults, type a name for the agent. + - If you want to create a configuration with CI/CD defaults, type a name that meets [the naming convention](#agent-naming-convention). - If you already have an [agent configuration file](#create-an-agent-configuration-file), select it from the list. 1. Select **Register an agent**. 1. GitLab generates an access token for the agent. Securely store this token. You need it to install the agent in your cluster and to [update the agent](#update-the-agent-version) to another version. @@ -63,24 +75,23 @@ To register an agent with GitLab: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. > - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. -The agent is configured through a configuration file. This file is optional. Without a configuration file, you can still use the CI/CD workflow in the project where the agent is registered. - -You need a configuration file if: +The agent uses a YAML file for configuration settings. You need a configuration file if: - You want to use [a GitOps workflow](../gitops.md#gitops-configuration-reference). -- You want to authorize a different project to use the agent for a [GitLab CI/CD workflow](../ci_cd_tunnel.md#authorize-the-agent). +- You want to authorize a different project to use the agent for a [GitLab CI/CD workflow](../ci_cd_workflow.md#authorize-the-agent). -To create an agent configuration file, go to the GitLab project. In the repository, create a file called `config.yaml` at this path: +To create an agent configuration file: -```plaintext -.gitlab/agents/<agent-name>/config.yaml -``` +1. In the repository, create a directory in this location. The `<agent-name>` must meet [the naming convention](#agent-naming-convention). + + ```plaintext + .gitlab/agents/<agent-name> + ``` -- Ensure the agent name follows the [naming convention](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name). -- Ensure the filename has the `.yaml` file extension (`config.yaml`). The `.yml` extension is not accepted. -- Add content to the `config.yaml` file: - - For a GitOps workflow, view [the configuration reference](../gitops.md#gitops-configuration-reference) for details. - - For a GitLab CI/CD workflow, you can leave the file blank for now. +1. In the directory, create a `config.yaml` file. Ensure the filename ends in `.yaml`, not `.yml`. +1. Add content to the `config.yaml` file: + - For a GitOps workflow, view [the configuration reference](../gitops.md#gitops-configuration-reference) for details. + - For a GitLab CI/CD workflow, view [the configuration reference](../ci_cd_workflow.md) for details. ### Install the agent in the cluster @@ -98,9 +109,9 @@ If you do not know which one to choose, we recommend starting with Helm. To install the agent on your cluster using Helm: -1. [Install Helm](https://helm.sh/docs/intro/install/) +1. [Install Helm](https://helm.sh/docs/intro/install/). 1. In your computer, open a terminal and [connect to your cluster](https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/). -1. Run the command you copied when registering your agent with GitLab. +1. Run the command you copied when you [registered your agent with GitLab](#register-the-agent-with-gitlab). Optionally, you can [customize the Helm installation](#customize-the-helm-installation). diff --git a/doc/user/clusters/agent/repository.md b/doc/user/clusters/agent/repository.md index 2087c804e26..8f3a8830202 100644 --- a/doc/user/clusters/agent/repository.md +++ b/doc/user/clusters/agent/repository.md @@ -1,210 +1,11 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'work_with_agent.md' +remove_date: '2022-07-19' --- -# Working with the agent for Kubernetes **(FREE)** +This document was moved to [another location](work_with_agent.md). -Use the following tasks when working with the agent for Kubernetes. - -## View your agents - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, the installed `agentk` version is displayed on the **Agent** tab. - -Prerequisite: - -- You must have at least the Developer role. - -To view the list of agents: - -1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. -1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. Select **Agent** tab to view clusters connected to GitLab through the agent. - -On this page, you can view: - -- All the registered agents for the current project. -- The connection status. -- The version of `agentk` installed on your cluster. -- The path to each agent configuration file. - -## View an agent's activity information - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/277323) in GitLab 14.6. - -The activity logs help you to identify problems and get the information -you need for troubleshooting. You can see events from a week before the -current date. To view an agent's activity: - -1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. -1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. Select the agent you want to see activity for. - -The activity list includes: - -- Agent registration events: When a new token is **created**. -- Connection events: When an agent is successfully **connected** to a cluster. - -The connection status is logged when you connect an agent for -the first time or after more than an hour of inactivity. - -View and provide feedback about the UI in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/4739). - -## Debug the agent - -To debug the cluster-side component (`agentk`) of the agent, set the log -level according to the available options: - -- `off` -- `warning` -- `error` -- `info` -- `debug` - -The log level defaults to `info`. You can change it by using a top-level `observability` -section in the configuration file, for example: - -```yaml -observability: - logging: - level: debug -``` - -## Reset the agent token - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327152) in GitLab 14.9. - -To reset the agent token without downtime: - -1. Create a new token: - 1. On the top bar, select **Menu > Projects** and find your project. - 1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. - 1. Select the agent you want to create a token for. - 1. On the **Tokens** tab, select **Create token**. - 1. Enter token's name and description (optional) and select **Create token**. -1. Securely store the generated token. -1. Use the token to [install the agent in your cluster](install/index.md#install-the-agent-in-the-cluster) and to [update the agent](install/index.md#update-the-agent-version) to another version. -1. Delete the token you're no longer using. - -## Remove an agent - -You can remove an agent by using the [GitLab UI](#remove-an-agent-through-the-gitlab-ui) or the -[GraphQL API](#remove-an-agent-with-the-gitlab-graphql-api). The agent and any associated tokens -are removed from GitLab, but no changes are made in your Kubernetes cluster. You must -clean up those resources manually. - -### Remove an agent through the GitLab UI - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323055) in GitLab 14.7. - -To remove an agent from the UI: - -1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file. -1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. -1. In the table, in the row for your agent, in the **Options** column, select the vertical ellipsis (**{ellipsis_v}**). -1. Select **Delete agent**. - -### Remove an agent with the GitLab GraphQL API - -1. Get the `<cluster-agent-token-id>` from a query in the interactive GraphQL explorer. - - For GitLab.com, go to <https://gitlab.com/-/graphql-explorer> to open GraphQL Explorer. - - For self-managed GitLab, go to `https://gitlab.example.com/-/graphql-explorer`, replacing `gitlab.example.com` with your instance's URL. - - ```graphql - query{ - project(fullPath: "<full-path-to-agent-configuration-project>") { - clusterAgent(name: "<agent-name>") { - id - tokens { - edges { - node { - id - } - } - } - } - } - } - ``` - -1. Remove an agent record with GraphQL by deleting the `clusterAgentToken`. - - ```graphql - mutation deleteAgent { - clusterAgentDelete(input: { id: "<cluster-agent-id>" } ) { - errors - } - } - - mutation deleteToken { - clusterAgentTokenDelete(input: { id: "<cluster-agent-token-id>" }) { - errors - } - } - ``` - -1. Verify whether the removal occurred successfully. If the output in the Pod logs includes `unauthenticated`, it means that the agent was successfully removed: - - ```json - { - "level": "warn", - "time": "2021-04-29T23:44:07.598Z", - "msg": "GetConfiguration.Recv failed", - "error": "rpc error: code = Unauthenticated desc = unauthenticated" - } - ``` - -1. Delete the agent in your cluster: - - ```shell - kubectl delete -n gitlab-kubernetes-agent -f ./resources.yml - ``` - -## Surface network security alerts from cluster to GitLab **(ULTIMATE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Cilium integration is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The agent for Kubernetes also provides an integration with Cilium. This integration provides a simple way to -generate network policy-related alerts and to surface those alerts in GitLab. - -Several components work in concert for the agent to generate the alerts: - -- A working Kubernetes cluster. -- Cilium integration through either of these options: - - Installation through [cluster management template](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium). - - Enablement of [hubble-relay](https://docs.cilium.io/en/v1.8/concepts/overview/#hubble) on an - existing installation. -- One or more network policies through any of these options: - - Use the [Container Network Policy editor](../../application_security/policies/index.md#container-network-policy-editor) to create and manage policies. - - Use an [AutoDevOps](../../application_security/policies/index.md#container-network-policy) configuration. - - Add the required labels and annotations to existing network policies. -- A configuration repository with [Cilium configured in `config.yaml`](repository.md#surface-network-security-alerts-from-cluster-to-gitlab) - -The setup process follows the same [agent's installation steps](install/index.md), -with the following differences: - -- When you define a configuration repository, you must do so with [Cilium settings](repository.md#surface-network-security-alerts-from-cluster-to-gitlab). -- You do not need to specify the `gitops` configuration section. - -To integrate, add a top-level `cilium` section to your `config.yml` file. Currently, the -only configuration option is the Hubble relay address: - -```yaml -cilium: - hubble_relay_address: "<hubble-relay-host>:<hubble-relay-port>" -``` - -If your Cilium integration was performed through [GitLab Managed Apps](../applications.md#install-cilium-using-gitlab-cicd) or the -[cluster management template](../../project/clusters/protect/container_network_security/quick_start_guide.md#use-the-cluster-management-template-to-install-cilium), -you can use `hubble-relay.gitlab-managed-apps.svc.cluster.local:80` as the address: - -```yaml -cilium: - hubble_relay_address: "hubble-relay.gitlab-managed-apps.svc.cluster.local:80" -``` +<!-- This redirect file can be deleted after <2022-07-19>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/clusters/agent/troubleshooting.md b/doc/user/clusters/agent/troubleshooting.md index c5c7e46c078..0932e9179f9 100644 --- a/doc/user/clusters/agent/troubleshooting.md +++ b/doc/user/clusters/agent/troubleshooting.md @@ -11,7 +11,7 @@ When you are using the GitLab agent for Kubernetes, you might experience issues You can start by viewing the service logs: ```shell -kubectl logs -f -l=app=gitlab-agent -n gitlab-kubernetes-agent +kubectl logs -f -l=app=gitlab-agent -n gitlab-agent ``` If you are a GitLab administrator, you can also view the [GitLab agent server logs](../../../administration/clusters/kas.md#troubleshooting). @@ -113,14 +113,14 @@ will be picked up automatically. For example, if your internal CA certificate is `myCA.pem`: ```plaintext -kubectl -n gitlab-kubernetes-agent create configmap ca-pemstore --from-file=myCA.pem +kubectl -n gitlab-agent create configmap ca-pemstore --from-file=myCA.pem ``` Then in `resources.yml`: ```yaml spec: - serviceAccountName: gitlab-kubernetes-agent + serviceAccountName: gitlab-agent containers: - name: agent image: "registry.gitlab.com/gitlab-org/cluster-integration/gitlab-agent/agentk:<version>" @@ -140,7 +140,7 @@ Then in `resources.yml`: volumes: - name: token-volume secret: - secretName: gitlab-kubernetes-agent-token + secretName: gitlab-agent-token - name: ca-pemstore-volume configMap: name: ca-pemstore diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md index 480b09ff2ab..69f5b1d9063 100644 --- a/doc/user/clusters/agent/vulnerabilities.md +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -34,80 +34,34 @@ You can use [cluster image scanning](../../application_security/cluster_image_sc to scan container images in your cluster for security vulnerabilities. To begin scanning all resources in your cluster, add a `starboard` -configuration block to your agent configuration file with no `filters`: +configuration block to your agent configuration with a `cadence` field +containing a CRON expression for when the scans will be run. ```yaml starboard: vulnerability_report: - filters: [] + cadence: '0 0 * * *' # Daily at 00:00 (Kubernetes cluster time) ``` -The namespaces that are able to be scanned depend on the [Starboard Operator install mode](https://aquasecurity.github.io/starboard/latest/operator/configuration/#install-modes). -By default, the Starboard Operator only scans resources in the `default` namespace. To change this -behavior, edit the `STARBOARD_OPERATOR` environment variable in the `starboard-operator` deployment -definition. +The `cadence` field is required. GitLab supports the following types of CRON syntax for the cadence field: -By adding filters, you can limit scans by: +- A daily cadence of once per hour at a specified hour, for example: `0 18 * * *` +- A weekly cadence of once per week on a specified day and at a specified hour, for example: `0 13 * * 0` -- Resource name -- Kind -- Container name -- Namespace +It is possible that other elements of the CRON syntax will work in the cadence field, however, GitLab does not officially test or support them. -```yaml -starboard: - vulnerability_report: - filters: - - namespaces: - - staging - - production - kinds: - - Deployment - - DaemonSet - containers: - - ruby - - postgres - - nginx - resources: - - my-app-name - - postgres - - ingress-nginx -``` - -A resource is scanned if the resource matches any of the given names and all of the given filter -types (`namespaces`, `kinds`, `containers`, `resources`). If a filter type is omitted, then all -names are scanned. In this example, a resource isn't scanned unless it has a container named `ruby`, -`postgres`, or `nginx`, and it's a `Deployment`: - -```yaml -starboard: - vulnerability_report: - filters: - - kinds: - - Deployment - containers: - - ruby - - postgres - - nginx -``` - -There is also a global `namespaces` field that applies to all filters: +By default, cluster image scanning will attempt to scan the workloads in all +namespaces for vulnerabilities. The `vulnerability_report` block has a `namespaces` +field which can be used to restrict which namespaces are scanned. For example, +if you would like to scan only the `development`, `staging`, and `production` +namespaces, you can use this configuration: ```yaml starboard: vulnerability_report: + cadence: '0 0 * * *' namespaces: - - production - filters: - - kinds: - - Deployment - - kinds: - - DaemonSet - resources: - - log-collector + - development + - staging + - production ``` - -In this example, the following resources are scanned: - -- All deployments (`Deployment`) in the `production` namespace. -- All daemon sets (`DaemonSet`) named `log-collector` in the `production` namespace. diff --git a/doc/user/clusters/agent/work_with_agent.md b/doc/user/clusters/agent/work_with_agent.md new file mode 100644 index 00000000000..8872ecf7ce5 --- /dev/null +++ b/doc/user/clusters/agent/work_with_agent.md @@ -0,0 +1,163 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Working with the agent for Kubernetes **(FREE)** + +Use the following tasks when working with the agent for Kubernetes. + +## View your agents + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, the installed `agentk` version is displayed on the **Agent** tab. + +Prerequisite: + +- You must have at least the Developer role. + +To view the list of agents: + +1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select **Agent** tab to view clusters connected to GitLab through the agent. + +On this page, you can view: + +- All the registered agents for the current project. +- The connection status. +- The version of `agentk` installed on your cluster. +- The path to each agent configuration file. + +## View an agent's activity information + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/277323) in GitLab 14.6. + +The activity logs help you to identify problems and get the information +you need for troubleshooting. You can see events from a week before the +current date. To view an agent's activity: + +1. On the top bar, select **Menu > Projects** and find the project that contains your agent configuration file. +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select the agent you want to see activity for. + +The activity list includes: + +- Agent registration events: When a new token is **created**. +- Connection events: When an agent is successfully **connected** to a cluster. + +The connection status is logged when you connect an agent for +the first time or after more than an hour of inactivity. + +View and provide feedback about the UI in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/4739). + +## Debug the agent + +To debug the cluster-side component (`agentk`) of the agent, set the log +level according to the available options: + +- `off` +- `warning` +- `error` +- `info` +- `debug` + +The log level defaults to `info`. You can change it by using a top-level `observability` +section in the configuration file, for example: + +```yaml +observability: + logging: + level: debug +``` + +## Reset the agent token + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327152) in GitLab 14.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336641) in GitLab 14.10, the agent token can be revoked from the UI. + +To reset the agent token without downtime: + +1. Create a new token: + 1. On the top bar, select **Menu > Projects** and find your project. + 1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. + 1. Select the agent you want to create a token for. + 1. On the **Access tokens** tab, select **Create token**. + 1. Enter token's name and description (optional) and select **Create token**. +1. Securely store the generated token. +1. Use the token to [install the agent in your cluster](install/index.md#install-the-agent-in-the-cluster) and to [update the agent](install/index.md#update-the-agent-version) to another version. +1. To delete the token you're no longer using, return to the token list and select **Revoke** (**{remove}**). + +## Remove an agent + +You can remove an agent by using the [GitLab UI](#remove-an-agent-through-the-gitlab-ui) or the +[GraphQL API](#remove-an-agent-with-the-gitlab-graphql-api). The agent and any associated tokens +are removed from GitLab, but no changes are made in your Kubernetes cluster. You must +clean up those resources manually. + +### Remove an agent through the GitLab UI + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323055) in GitLab 14.7. + +To remove an agent from the UI: + +1. On the top bar, select **Menu > Projects** and find the project that contains the agent configuration file. +1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. In the table, in the row for your agent, in the **Options** column, select the vertical ellipsis (**{ellipsis_v}**). +1. Select **Delete agent**. + +### Remove an agent with the GitLab GraphQL API + +1. Get the `<cluster-agent-token-id>` from a query in the interactive GraphQL explorer. + - For GitLab.com, go to <https://gitlab.com/-/graphql-explorer> to open GraphQL Explorer. + - For self-managed GitLab, go to `https://gitlab.example.com/-/graphql-explorer`, replacing `gitlab.example.com` with your instance's URL. + + ```graphql + query{ + project(fullPath: "<full-path-to-agent-configuration-project>") { + clusterAgent(name: "<agent-name>") { + id + tokens { + edges { + node { + id + } + } + } + } + } + } + ``` + +1. Remove an agent record with GraphQL by deleting the `clusterAgentToken`. + + ```graphql + mutation deleteAgent { + clusterAgentDelete(input: { id: "<cluster-agent-id>" } ) { + errors + } + } + + mutation deleteToken { + clusterAgentTokenDelete(input: { id: "<cluster-agent-token-id>" }) { + errors + } + } + ``` + +1. Verify whether the removal occurred successfully. If the output in the Pod logs includes `unauthenticated`, it means that the agent was successfully removed: + + ```json + { + "level": "warn", + "time": "2021-04-29T23:44:07.598Z", + "msg": "GetConfiguration.Recv failed", + "error": "rpc error: code = Unauthenticated desc = unauthenticated" + } + ``` + +1. Delete the agent in your cluster: + + ```shell + kubectl delete -n gitlab-kubernetes-agent -f ./resources.yml + ``` diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 2bcfea50ee3..7b5ea7d12fd 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -2,1134 +2,12 @@ stage: Configure group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: '../../update/removals.md#managed-cluster-applicationsgitlab-ciyml' --- -# GitLab Managed Apps (DEPRECATED) **(FREE)** +# GitLab Managed Apps (removed) **(FREE)** -NOTE: -The new recommended way to manage cluster applications is to use the [cluster management project template](management_project_template.md). -If you want to migrate your GitLab managed apps management to this template, reference to [migrating from GitLab managed apps to project template](migrating_from_gma_to_project_template.md). - -**GitLab Managed Apps** was created to help you configure applications in your -cluster directly from GitLab. You could use this feature through two different -methods: "one-click install" and "CI/CD template". Both methods are **deprecated**: - -- The **one-click install** method was [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63348) in GitLab 14.0. -- The **CI/CD template method** was deprecated in GitLab 13.12 and is scheduled - to be removed in GitLab 15.0. - -Both methods were limiting as you couldn't fully customize your third-party apps installed -through GitLab Managed Apps. Therefore, we decided to deprecate this feature and provide -better [GitOps-driven alternatives](https://about.gitlab.com/direction/configure/kubernetes_management/#gitlab-managed-applications) to our users, such as [cluster integrations](integrations.md) and [cluster management project](management_project.md). - -## Install using GitLab CI/CD (DEPRECATED) - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20822) in GitLab 12.6. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -WARNING: -The GitLab Managed Apps CI/CD installation method was [deprecated in 13.12](https://gitlab.com/gitlab-org/gitlab/-/issues/327908). -Your applications continue to work. However, we no longer support and maintain the GitLab CI/CD template for -Managed Apps (`Managed-Cluster-Applications.gitlab-ci.yml`). -The new recommended way to manage cluster applications is to use the [cluster management project template](management_project_template.md). -If you want to migrate your GitLab managed apps management to this template, reference to [migrating from GitLab managed apps to project template](migrating_from_gma_to_project_template.md). - -The CI/CD template was the primary method for installing applications to clusters via GitLab Managed Apps -and customize them through Helm. - -Supported applications: - -- [Ingress](#install-ingress-using-gitlab-cicd) -- [cert-manager](#install-cert-manager-using-gitlab-cicd) -- [Sentry](#install-sentry-using-gitlab-cicd) -- [GitLab Runner](#install-gitlab-runner-using-gitlab-cicd) -- [Cilium](#install-cilium-using-gitlab-cicd) -- [Falco](#install-falco-using-gitlab-cicd) -- [Vault](#install-vault-using-gitlab-cicd) -- [JupyterHub](#install-jupyterhub-using-gitlab-cicd) -- [Elastic Stack](#install-elastic-stack-using-gitlab-cicd) -- [Crossplane](#install-crossplane-using-gitlab-cicd) -- [Fluentd](#install-fluentd-using-gitlab-cicd) -- [Knative](#install-knative-using-gitlab-cicd) -- [PostHog](#install-posthog-using-gitlab-cicd) -- [Prometheus](#install-prometheus-using-gitlab-cicd) - -### Prerequisites - -You can find and import all the files referenced below -in the [example cluster applications -project](https://gitlab.com/gitlab-org/cluster-integration/example-cluster-applications/). - -To install applications using GitLab CI/CD: - -1. Connect the cluster to a [cluster management project](management_project.md). -1. In that project, add a `.gitlab-ci.yml` file with the following content: - - ```yaml - include: - - template: Managed-Cluster-Applications.gitlab-ci.yml - ``` - - The job provided by this template connects to the `*` (default) cluster using tools provided - in a custom Docker image. It requires that you have a runner registered with the Docker, - Kubernetes, or Docker Machine executor. - - To install to a specific cluster, read - [Use the template with a custom environment](#use-the-template-with-a-custom-environment). - -1. Add a `.gitlab/managed-apps/config.yaml` file to define which - applications you would like to install. Define the `installed` key as - `true` to install the application and `false` to uninstall the - application. For example, to install Ingress: - - ```yaml - ingress: - installed: true - ``` - -1. Optionally, define `.gitlab/managed-apps/<application>/values.yaml` file to - customize values for the installed application. - -A GitLab CI/CD pipeline runs on the default branch to install the -applications you have configured. In case of pipeline failure, the -output of the [Helm Tiller](https://v2.helm.sh/docs/install/#running-tiller-locally) binary -is saved as a [CI job artifact](../../ci/pipelines/job_artifacts.md). - -#### Prerequisites in GitLab versions earlier than 13.5 - -For GitLab versions 13.5 and earlier, the Ingress, Fluentd, Prometheus, and Sentry -apps were fetched from the central Helm stable repository (`https://kubernetes-charts.storage.googleapis.com/`). -This repository [was deleted](https://github.com/helm/charts#deprecation-timeline) -on November 13, 2020. This causes the installation CI/CD pipeline to -fail. Upgrade to GitLab 13.6, or alternatively, you can -use the following `.gitlab-ci.yml`, which has been tested in GitLab 13.5: - -```yaml -include: - - template: Managed-Cluster-Applications.gitlab-ci.yml - -apply: - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" -``` - -### Use the template with a custom environment - -If you only want apps to be installed on a specific cluster, or if your cluster's -scope does not match `production`, you can override the environment name in your `.gitlab-ci.yml` -file: - -```yaml -include: - - template: Managed-Cluster-Applications.gitlab-ci.yml - -apply: - except: - variables: - - '$CI_JOB_NAME == "apply"' - -.managed-apps: - extends: apply - -example-install: - extends: .managed-apps - environment: - name: example/production -``` - -### Important notes - -Note the following: - -- We recommend using the cluster management project exclusively for managing deployments to a cluster. - Do not add your application's source code to such projects. -- When you set the value for `installed` key back to `false`, the application is - unprovisioned from the cluster. -- If you update `.gitlab/managed-apps/<application>/values.yaml` with new values, the - application is redeployed. - -### Install Ingress using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -To install Ingress, define the `.gitlab/managed-apps/config.yaml` file -with: - -```yaml -ingress: - installed: true -``` - -Ingress is installed into the `gitlab-managed-apps` namespace -of your cluster. - -You can customize the installation of Ingress by defining a -`.gitlab/managed-apps/ingress/values.yaml` file in your cluster -management project. Refer to the -[chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress) -for the available configuration options. - -Support for installing the Ingress managed application is provided by the GitLab Configure group. -If you run into unknown issues, [open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), -and ping at least 2 people from the -[Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). - -### Install cert-manager using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -cert-manager is installed using GitLab CI/CD by defining configuration in -`.gitlab/managed-apps/config.yaml`. - -cert-manager: - -- Is installed into the `gitlab-managed-apps` namespace of your cluster. -- Can be installed with or without a default - [Let's Encrypt `ClusterIssuer`](https://cert-manager.io/docs/configuration/acme/), which requires an - email address to be specified. The email address is used by Let's Encrypt to - contact you about expiring certificates and issues related to your account. - -The following configuration is required to install cert-manager using GitLab CI/CD: - -```yaml -certManager: - installed: true - letsEncryptClusterIssuer: - installed: true - email: "user@example.com" -``` - -The following installs cert-manager using GitLab CI/CD without the default `ClusterIssuer`: - -```yaml -certManager: - installed: true - letsEncryptClusterIssuer: - installed: false -``` - -You can customize the installation of cert-manager by defining a -`.gitlab/managed-apps/cert-manager/values.yaml` file in your cluster -management project. Refer to the -[chart](https://github.com/jetstack/cert-manager) for the -available configuration options. - -Support for installing the Cert Manager managed application is provided by the -GitLab Configure group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). - -### Install Sentry using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -The Sentry Helm chart [recommends](https://github.com/helm/charts/blob/f6e5784f265dd459c5a77430185d0302ed372665/stable/sentry/values.yaml#L284-L285) -at least 3 GB of available RAM for database migrations. - -To install Sentry, define the `.gitlab/managed-apps/config.yaml` file -with: - -```yaml -sentry: - installed: true -``` - -Sentry is installed into the `gitlab-managed-apps` namespace -of your cluster. - -You can customize the installation of Sentry by defining -`.gitlab/managed-apps/sentry/values.yaml` file in your cluster -management project. Refer to the -[chart](https://github.com/helm/charts/tree/master/stable/sentry) -for the available configuration options. - -We recommend you pay close attention to the following configuration options: - -- `email`. Needed to invite users to your Sentry instance and to send error emails. -- `user`. Where you can set the login credentials for the default administrator user. -- `postgresql`. For a PostgreSQL password that can be used when running future updates. - -When upgrading, it's important to provide the existing PostgreSQL password (given -using the `postgresql.postgresqlPassword` key) to avoid authentication errors. -Read the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) -for more information. - -Here is an example configuration for Sentry: - -```yaml -# Admin user to create -user: - # Indicated to create the admin user or not, - # Default is true as the initial installation. - create: true - email: "<your email>" - password: "<your password>" - -email: - from_address: "<your from email>" - host: smtp - port: 25 - use_tls: false - user: "<your email username>" - password: "<your email password>" - enable_replies: false - -ingress: - enabled: true - hostname: "<sentry.example.com>" - -# Needs to be here between runs. -# See https://github.com/helm/charts/tree/master/stable/postgresql#upgrade for more info -postgresql: - postgresqlPassword: example-postgresql-password -``` - -Support for installing the Sentry managed application is provided by the -GitLab Monitor group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Monitor group](https://about.gitlab.com/handbook/product/categories/#monitor-group). - -### Install PostHog using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -[PostHog](https://posthog.com) 🦔 is a developer-friendly, open-source product analytics platform. - -To install PostHog into the `gitlab-managed-apps` namespace of your cluster, -define the `.gitlab/managed-apps/config.yaml` file with: - -```yaml -posthog: - installed: true -``` - -You can customize the installation of PostHog by defining `.gitlab/managed-apps/posthog/values.yaml` -in your cluster management project. Refer to the -[Configuration section](https://github.com/PostHog/charts/tree/master/charts/posthog) -of the PostHog chart's README for the available configuration options. - -You must provide a PostgreSQL password in `postgresql.postgresqlPassword` -to avoid authentication errors. Read the -[PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) -for more information. - -Redis pods are restarted between upgrades. To prevent downtime, provide a Redis -password using the `redis.password` key. This prevents a new password from -being generated on each restart. - -Here is an example configuration for PostHog: - -```yaml -ingress: - enabled: true - hostname: "<posthog.example.com>" - -# This will be autogenerated if you skip it. Include if you have 2 or more web replicas -posthogSecret: 'long-secret-key-used-to-sign-cookies' - -# Needs to be here between runs. -# See https://github.com/helm/charts/tree/master/stable/postgresql#upgrade for more info -postgresql: - postgresqlPassword: example-postgresql-password - -# Recommended to set this to a value to redis prevent downtime between upgrades -redis: - password: example-redis-password -``` - -Support for the PostHog managed application is provided by the PostHog team. -If you run into issues, -[open a support ticket](https://github.com/PostHog/posthog/issues/new/choose) directly. - -### Install Prometheus using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25138) in GitLab 12.8. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -[Prometheus](https://prometheus.io/docs/introduction/overview/) is an -open-source monitoring and alerting system for supervising your -deployed applications. - -To install Prometheus into the `gitlab-managed-apps` namespace of your cluster, -define the `.gitlab/managed-apps/config.yaml` file with: - -```yaml -prometheus: - installed: true -``` - -You can customize the installation of Prometheus by defining -`.gitlab/managed-apps/prometheus/values.yaml` in your cluster management -project. Refer to the -[Configuration section](https://github.com/helm/charts/tree/master/stable/prometheus#configuration) -of the Prometheus chart's README for the available configuration options. - -Support for installing the Prometheus managed application is provided by the -GitLab Monitor group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the [Monitor group](https://about.gitlab.com/handbook/product/categories/#monitor-group). - -### Install GitLab Runner using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -GitLab Runner is installed using GitLab CI/CD by defining configuration in -`.gitlab/managed-apps/config.yaml`. - -The following configuration is required to install GitLab Runner using GitLab CI/CD: - -```yaml -gitlabRunner: - installed: true -``` - -GitLab Runner is installed into the `gitlab-managed-apps` namespace of your cluster. - -For GitLab Runner to function, you _must_ specify the following: - -- `gitlabUrl`: The GitLab server full URL (for example, `https://gitlab.example.com`) - to register the Runner against. -- `runnerRegistrationToken`: The registration token for adding new runners to GitLab. - This must be [retrieved from your GitLab instance](../../ci/runners/index.md). - -These values can be specified using [CI/CD variables](../../ci/variables/index.md): - -- `GITLAB_RUNNER_GITLAB_URL` is used for `gitlabUrl`. -- `GITLAB_RUNNER_REGISTRATION_TOKEN` is used for `runnerRegistrationToken` - -The methods of specifying these values are mutually exclusive. Either specify variables `GITLAB_RUNNER_REGISTRATION_TOKEN` and `GITLAB_RUNNER_TOKEN` as CI variables (recommended) or provide values for `runnerRegistrationToken:` and `runnerToken:` in `.gitlab/managed-apps/gitlab-runner/values.yaml`. If you choose to use CI variables, comment out or remove `runnerRegistrationToken:` and `runnerToken:` from `.gitlab/managed-apps/gitlab-runner/values`. - -The runner registration token allows connection to a project by a runner and therefore should be treated as a secret to prevent malicious use and code exfiltration through a runner. For this reason, we recommend that you specify the runner registration token as a [protected variable](../../ci/variables/index.md#protect-a-cicd-variable) and [masked variable](../../ci/variables/index.md#mask-a-cicd-variable) and do not commit them to the Git repository in the `values.yaml` file. - -You can customize the installation of GitLab Runner by defining -`.gitlab/managed-apps/gitlab-runner/values.yaml` file in your cluster -management project. Refer to the -[chart](https://gitlab.com/gitlab-org/charts/gitlab-runner) for the -available configuration options. - -Support for installing the GitLab Runner managed application is provided by the -GitLab Runner group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Runner group](https://about.gitlab.com/handbook/product/categories/#runner-group). - -### Install Cilium using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/22) in GitLab 12.8. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -[Cilium](https://cilium.io/) is a networking plugin for Kubernetes that you can use to implement -support for [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -resources. For more information, see [Network Policies](../../topics/autodevops/stages.md#network-policy). - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview, see the -[Container Network Security Demo for GitLab 12.8](https://www.youtube.com/watch?v=pgUEdhdhoUI). - -Enable Cilium in the `.gitlab/managed-apps/config.yaml` file to install it: - -```yaml -# possible values are gke or eks -clusterType: gke - -cilium: - installed: true -``` - -The `clusterType` variable enables the recommended Helm variables for a corresponding cluster type. -You can check the recommended variables for each cluster type in the official documentation: - -- [Google GKE](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-gke/#deploy-cilium) -- [AWS EKS](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-eks/#deploy-cilium) - -Do not use `clusterType` for sandbox environments like [minikube](https://minikube.sigs.k8s.io/docs/). - -You can customize Cilium's Helm variables by defining the -`.gitlab/managed-apps/cilium/values.yaml` file in your cluster -management project. Refer to the -[Cilium chart](https://github.com/cilium/cilium/tree/master/install/kubernetes/cilium) -for the available configuration options. - -You can check Cilium's installation status on the cluster management page: - -- [Project-level cluster](../project/clusters/index.md): Navigate to your project's - **Infrastructure > Kubernetes clusters** page. -- [Group-level cluster](../group/clusters/index.md): Navigate to your group's - **Kubernetes** page. - -WARNING: -Installation and removal of the Cilium requires a **manual** -[restart](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-helm/#restart-unmanaged-pods) -of all affected pods in all namespaces to ensure that they are -[managed](https://docs.cilium.io/en/v1.8/operations/troubleshooting/#ensure-managed-pod) -by the correct networking plugin. Whenever Hubble is enabled, its related pod might require a -restart depending on whether it started prior to Cilium. For more information, see -[Failed Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#failed-deployment) -in the Kubernetes docs. - -NOTE: -Major upgrades might require additional setup steps. For more information, see -the official [upgrade guide](https://docs.cilium.io/en/v1.8/operations/upgrade/). - -By default, Cilium's -[audit mode](https://docs.cilium.io/en/v1.8/gettingstarted/policy-creation/#enable-policy-audit-mode) -is enabled. In audit mode, Cilium doesn't drop disallowed packets. You -can use `policy-verdict` log to observe policy-related decisions. You -can disable audit mode by adding the following to -`.gitlab/managed-apps/cilium/values.yaml`: - -```yaml -config: - policyAuditMode: false - -agent: - monitor: - eventTypes: ["drop"] # Note: possible values are documented at https://docs.cilium.io/en/stable/cmdref/cilium_monitor/ -``` - -The Cilium monitor log for traffic is logged out by the -`cilium-monitor` sidecar container. You can check these logs with the following command: - -```shell -kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor -``` - -You can disable the monitor log in `.gitlab/managed-apps/cilium/values.yaml`: - -```yaml -agent: - monitor: - enabled: false -``` - -The [Hubble](https://github.com/cilium/hubble) monitoring daemon is enabled by default -and it's set to collect per namespace flow metrics. This metrics are accessible on the -[Threat Monitoring](../application_security/threat_monitoring/index.md) -dashboard. You can disable Hubble by adding the following to -`.gitlab/managed-apps/cilium/values.yaml`: - -```yaml -global: - hubble: - enabled: false -``` - -You can also adjust Helm values for Hubble by using -`.gitlab/managed-apps/cilium/values.yaml`: - -```yaml -global: - hubble: - enabled: true - metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' -``` - -Support for installing the Cilium managed application is provided by the -GitLab Container Security group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Container Security group](https://about.gitlab.com/handbook/product/categories/#container-security-group). - -### Install Falco using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/91) in GitLab 13.1. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -GitLab Container Host Security Monitoring uses [Falco](https://falco.org/) -as a runtime security tool that listens to the Linux kernel using eBPF. Falco parses system calls -and asserts the stream against a configurable rules engine in real-time. For more information, see -[Falco's Documentation](https://falco.org/docs/). - -You can enable Falco in the -`.gitlab/managed-apps/config.yaml` file: - -```yaml -falco: - installed: true -``` - -You can customize Falco's Helm variables by defining the -`.gitlab/managed-apps/falco/values.yaml` file in your cluster -management project. Refer to the -[Falco chart](https://github.com/falcosecurity/charts/tree/master/falco) -for the available configuration options. - -WARNING: -By default eBPF support is enabled and Falco uses an -[eBPF probe](https://falco.org/docs/event-sources/drivers/#using-the-ebpf-probe) -to pass system calls to user space. If your cluster doesn't support this, you can -configure it to use Falco kernel module instead by adding the following to -`.gitlab/managed-apps/falco/values.yaml`: - -```yaml -ebpf: - enabled: false -``` - -In rare cases where probe installation on your cluster isn't possible and the kernel/probe -isn't pre-compiled, you may need to manually prepare the kernel module or eBPF probe with -[`driverkit`](https://github.com/falcosecurity/driverkit#against-a-kubernetes-cluster) -and install it on each cluster node. - -By default, Falco is deployed with a limited set of rules. To add more rules, add -the following to `.gitlab/managed-apps/falco/values.yaml` (you can get examples from -[Cloud Native Security Hub](https://securityhub.dev/)): - -```yaml -customRules: - file-integrity.yaml: |- - - rule: Detect New File - desc: detect new file created - condition: > - evt.type = chmod or evt.type = fchmod - output: > - File below a known directory opened for writing (user=%user.name - command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) - priority: ERROR - tags: [filesystem] - - rule: Detect New Directory - desc: detect new directory created - condition: > - mkdir - output: > - File below a known directory opened for writing (user=%user.name - command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) - priority: ERROR - tags: [filesystem] -``` - -By default, Falco only outputs security events to logs as JSON objects. To set it to output to an -[external API](https://falco.org/docs/alerts/#https-output-send-alerts-to-an-https-end-point) -or [application](https://falco.org/docs/alerts/#program-output), -add the following to `.gitlab/managed-apps/falco/values.yaml`: - -```yaml -falco: - programOutput: - enabled: true - keepAlive: false - program: mail -s "Falco Notification" someone@example.com - - httpOutput: - enabled: true - url: http://some.url -``` - -You can check these logs with the following command: - -```shell -kubectl -n gitlab-managed-apps logs -l app=falco -``` - -Support for installing the Falco managed application is provided by the -GitLab Container Security group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Container Security group](https://about.gitlab.com/handbook/product/categories/#container-security-group). - -### Install Vault using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9982) in GitLab 12.9. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -[HashiCorp Vault](https://www.vaultproject.io/) is a secrets management solution which -can be used to safely manage and store passwords, credentials, certificates, and more. A Vault -installation could be leveraged to provide a single secure data store for credentials -used in your applications, GitLab CI/CD jobs, and more. It could also serve as a way of -providing SSL/TLS certificates to systems and deployments in your infrastructure. Leveraging -Vault as a single source for all these credentials allows greater security by having -a single source of access, control, and auditability around all your sensitive -credentials and certificates. This feature requires giving GitLab the highest level of access and -control. Therefore, if GitLab is compromised, the security of this Vault instance is as well. To -avoid this security risk, GitLab recommends using your own HashiCorp Vault to leverage -[external secrets with CI](../../ci/secrets/index.md). - -To install Vault, enable it in the `.gitlab/managed-apps/config.yaml` file: - -```yaml -vault: - installed: true -``` - -By default you receive a basic Vault setup with no scalable storage backend. This -is enough for simple testing and small-scale deployments, though has limits -to how much it can scale, and as it's a single instance deployment, upgrading the -Vault application causes downtime. - -To optimally use Vault in a production environment, it's ideal to have a good understanding -of the internals of Vault and how to configure it. This can be done by reading -the [Vault Configuration guide](../../ci/secrets/#configure-your-vault-server), -the [Vault documentation](https://www.vaultproject.io/docs/internals) and -the Vault Helm chart [`values.yaml` file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). - -At a minimum, most users set up: - -- A [seal](https://www.vaultproject.io/docs/configuration/seal) for extra encryption - of the main key. -- A [storage backend](https://www.vaultproject.io/docs/configuration/storage) that's - suitable for environment and storage security requirements. -- [HA Mode](https://www.vaultproject.io/docs/concepts/ha). -- The [Vault UI](https://www.vaultproject.io/docs/configuration/ui). - -The following is an example values file (`.gitlab/managed-apps/vault/values.yaml`) -that configures Google Key Management Service for auto-unseal, using a Google Cloud Storage backend, enabling -the Vault UI, and enabling HA with 3 pod replicas. The `storage` and `seal` stanzas -below are examples and should be replaced with settings specific to your environment. - -```yaml -# Enable the Vault WebUI -ui: - enabled: true -server: - # Disable the built in data storage volume as it's not safe for High Availability mode - dataStorage: - enabled: false - # Enable High Availability Mode - ha: - enabled: true - # Configure Vault to listen on port 8200 for normal traffic and port 8201 for inter-cluster traffic - config: | - listener "tcp" { - tls_disable = 1 - address = "[::]:8200" - cluster_address = "[::]:8201" - } - # Configure Vault to store its data in a GCS Bucket backend - storage "gcs" { - path = "gcs://my-vault-storage/vault-bucket" - ha_enabled = "true" - } - # Configure Vault to unseal storage using a GKMS key - seal "gcpckms" { - project = "vault-helm-dev-246514" - region = "global" - key_ring = "vault-helm-unseal-kr" - crypto_key = "vault-helm-unseal-key" - } -``` - -After you have successfully installed Vault, you must -[initialize the Vault](https://learn.hashicorp.com/tutorials/vault/getting-started-deploy#initializing-the-vault) -and obtain the initial root token. You need access to your Kubernetes cluster that -Vault has been deployed into in order to do this. To initialize the Vault, get a -shell to one of the Vault pods running inside Kubernetes (typically this is done -by using the `kubectl` command line tool). After you have a shell into the pod, -run the `vault operator init` command: - -```shell -kubectl -n gitlab-managed-apps exec -it vault-0 sh -/ $ vault operator init -``` - -This should give you your unseal keys and initial root token. Make sure to note these down -and keep these safe, as they're required to unseal the Vault throughout its lifecycle. - -Support for installing the Vault managed application is provided by the -GitLab Release Management group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Release Management group](https://about.gitlab.com/handbook/product/categories/#release-management-group). - -### Install JupyterHub using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/40) in GitLab 12.8. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -JupyterHub is installed using GitLab CI/CD by defining configuration in -`.gitlab/managed-apps/config.yaml` as follows: - -```yaml -jupyterhub: - installed: true - gitlabProjectIdWhitelist: [] - gitlabGroupWhitelist: [] -``` - -In the configuration: - -- `gitlabProjectIdWhitelist` restricts GitLab authentication to only members of the specified projects. -- `gitlabGroupWhitelist` restricts GitLab authentication to only members of the specified groups. -- Specifying an empty array for both allows any user on the GitLab instance to sign in. - -JupyterHub is installed into the `gitlab-managed-apps` namespace of your cluster. - -For JupyterHub to function, you must set up an [OAuth Application](../../integration/oauth_provider.md). -Set: - -- "Redirect URI" to `http://<JupyterHub Host>/hub/oauth_callback`. -- "Scope" to `api read_repository write_repository`. - -In addition, the following variables must be specified using [CI/CD variables](../../ci/variables/index.md): - -- `JUPYTERHUB_PROXY_SECRET_TOKEN` - Secure string used for signing communications - from the hub. Read [`proxy.secretToken`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#proxy-secrettoken). -- `JUPYTERHUB_COOKIE_SECRET` - Secure string used for signing secure cookies. Read - [`hub.cookieSecret`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#hub-cookiesecret). -- `JUPYTERHUB_HOST` - Hostname used for the installation. For example, `jupyter.gitlab.example.com`. -- `JUPYTERHUB_GITLAB_HOST` - Hostname of the GitLab instance used for authentication. - For example, `gitlab.example.com`. -- `JUPYTERHUB_AUTH_CRYPTO_KEY` - A 32-byte encryption key used to set - [`auth.state.cryptoKey`](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html#auth-state-cryptokey). -- `JUPYTERHUB_AUTH_GITLAB_CLIENT_ID` - "Application ID" for the OAuth Application. -- `JUPYTERHUB_AUTH_GITLAB_CLIENT_SECRET` - "Secret" for the OAuth Application. - -By default, JupyterHub is installed using a -[default values file](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/blob/master/src/default-data/jupyterhub/values.yaml.gotmpl). -You can customize the installation of JupyterHub by defining a -`.gitlab/managed-apps/jupyterhub/values.yaml` file in your cluster management project. - -Refer to the -[chart reference](https://zero-to-jupyterhub.readthedocs.io/en/stable/reference/reference.html) for the -available configuration options. - -Support for installing the JupyterHub managed application is provided by the GitLab Configure group. -If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). - -### Install Elastic Stack using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25138) in GitLab 12.8. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -Elastic Stack is installed using GitLab CI/CD by defining configuration in -`.gitlab/managed-apps/config.yaml`. - -The following configuration is required to install Elastic Stack using GitLab CI/CD: - -```yaml -elasticStack: - installed: true -``` - -Elastic Stack is installed into the `gitlab-managed-apps` namespace of your cluster. - -You can check the default -[`values.yaml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/elastic_stack/values.yaml) -we set for this chart. - -You can customize the installation of Elastic Stack by defining -`.gitlab/managed-apps/elastic-stack/values.yaml` file in your cluster -management project. Refer to the -[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for all -available configuration options. - -Support for installing the Elastic Stack managed application is provided by the -GitLab Monitor group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the [Monitor group](https://about.gitlab.com/handbook/product/categories/#monitor-group). - -### Install Crossplane using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35675) in GitLab 12.9. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -Crossplane is installed using GitLab CI/CD by defining configuration in -`.gitlab/managed-apps/config.yaml`. - -The following configuration is required to install Crossplane using GitLab CI/CD: - -```yaml -Crossplane: - installed: true -``` - -Crossplane is installed into the `gitlab-managed-apps` namespace of your cluster. - -You can check the default -[`values.yaml`](https://github.com/crossplane/crossplane/blob/master/cluster/charts/crossplane/values.yaml.tmpl) -we set for this chart. - -You can customize the installation of Crossplane by defining -`.gitlab/managed-apps/crossplane/values.yaml` file in your cluster -management project. Refer to the -[chart](https://github.com/crossplane/crossplane/tree/master/cluster/charts/crossplane#configuration) -for the available configuration options. Note that this link points to the documentation -for the current development release, which may differ from the version you have installed. - -Support for the Crossplane managed application is provided by the Crossplane team. -If you run into issues, -[open a support ticket](https://github.com/crossplane/crossplane/issues/new/choose) directly. - -### Install Fluentd using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/76) in GitLab 12.10. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -To install Fluentd into the `gitlab-managed-apps` namespace of your cluster using -GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: - -```yaml -Fluentd: - installed: true -``` - -You can also review the default values set for this chart in the -[`values.yaml`](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. - -You can customize the installation of Fluentd by defining -`.gitlab/managed-apps/fluentd/values.yaml` file in your cluster management -project. Refer to the -[configuration chart](https://github.com/helm/charts/tree/master/stable/fluentd#configuration) -for the current development release of Fluentd for all available configuration options. - -The configuration chart link points to the current development release, which -may differ from the version you have installed. To ensure compatibility, switch -to the specific branch or tag you are using. - -Support for installing the Fluentd managed application is provided by the -GitLab Container Security group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Container Security group](https://about.gitlab.com/handbook/product/categories/#container-security-group). - -### Install Knative using GitLab CI/CD - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -To install Knative, define the `.gitlab/managed-apps/config.yaml` file -with: - -```yaml -knative: - installed: true -``` - -You can customize the installation of Knative by defining `.gitlab/managed-apps/knative/values.yaml` -file in your cluster management project. Refer to the [chart](https://gitlab.com/gitlab-org/charts/knative) -for all available configuration options. - -Here is an example configuration for Knative: - -```yaml -domain: 'my.wildcard.A.record.dns' -``` - -If you plan to use GitLab Serverless capabilities, be sure to set an `A record` -wildcard domain on your custom configuration. - -Support for installing the Knative managed application is provided by the -GitLab Configure group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Configure group](https://about.gitlab.com/handbook/product/categories/#configure-group). - -#### Knative Metrics - -GitLab provides [Invocation Metrics](../project/clusters/serverless/index.md#invocation-metrics) -for your functions. To collect these metrics, you must have: - -1. Knative and Prometheus managed applications installed on your cluster. -1. Manually applied the custom metrics on your cluster by running the following command: - - ```shell - kubectl apply -f https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/raw/02c8231e30ef5b6725e6ba368bc63863ceb3c07d/src/default-data/knative/istio-metrics.yaml - ``` - -#### Uninstall Knative - -To uninstall Knative, you must first manually remove any custom metrics you have added -by running the following command: - -```shell -kubectl delete -f https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/raw/02c8231e30ef5b6725e6ba368bc63863ceb3c07d/src/default-data/knative/istio-metrics.yaml -``` - -### Install AppArmor using GitLab CI/CD - -> - [Introduced](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications/-/merge_requests/100) in GitLab 13.1. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. - -To install AppArmor into the `gitlab-managed-apps` namespace of your cluster using -GitLab CI/CD, define the following configuration in `.gitlab/managed-apps/config.yaml`: - -```yaml -apparmor: - installed: true -``` - -You can define one or more AppArmor profiles by adding them into -`.gitlab/managed-apps/apparmor/values.yaml` as the following: - -```yaml -profiles: - profile-one: |- - profile profile-one { - file, - } -``` - -Refer to the [AppArmor chart](https://gitlab.com/gitlab-org/charts/apparmor) for more information on this chart. - -#### Using AppArmor profiles in your deployments - -After installing AppAmor, you can use profiles by adding Pod Annotations. If you're using -Auto DevOps, you can [customize `auto-deploy-values.yaml`](../../topics/autodevops/customize.md#customize-values-for-helm-chart) -to annotate your pods. Although it's helpful to be aware of the -[list of custom attributes](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app#gitlabs-auto-deploy-helm-chart), -you're only required to set `podAnnotations` as follows: - -```yaml -podAnnotations: - container.apparmor.security.beta.kubernetes.io/auto-deploy-app: localhost/profile-one -``` - -The only information to be changed here is the profile name which is `profile-one` -in this example. Refer to the -[AppArmor tutorial](https://kubernetes.io/docs/tutorials/security/apparmor/#securing-a-pod) -for more information on how AppArmor is integrated in Kubernetes. - -#### Using PodSecurityPolicy in your deployments - -To enable AppArmor annotations on a Pod Security Policy you must first -load the corresponding AppArmor profile. - -[Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) are -resources at the cluster level that control security-related -properties of deployed pods. You can use such a policy to enable -loaded AppArmor profiles and apply necessary pod restrictions across a -cluster. You can deploy a new policy by adding the following -to`.gitlab/managed-apps/apparmor/values.yaml`: - -```yaml -securityPolicies: - example: - defaultProfile: profile-one - allowedProfiles: - - profile-one - - profile-two - spec: - privileged: false - seLinux: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - runAsUser: - rule: RunAsAny - fsGroup: - rule: RunAsAny - volumes: - - '*' -``` - -This example creates a single policy named `example` with the provided specification, -and enables [AppArmor annotations](https://kubernetes.io/docs/tutorials/security/apparmor/#podsecuritypolicy-annotations) on it. - -Support for installing the AppArmor managed application is provided by the -GitLab Container Security group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping -at least 2 people from the -[Container Security group](https://about.gitlab.com/handbook/product/categories/#container-security-group). - -## Install with one click (REMOVED) - -> [Removed](https://gitlab.com/groups/gitlab-org/-/epics/4280) in GitLab 14.0. - -The one-click installation method was deprecated in GitLab 13.9 and removed in [GitLab 14.0](https://gitlab.com/groups/gitlab-org/-/epics/4280). -The removal does not break nor uninstall any apps you have installed, it only -removes the "Applications" tab from the cluster page. -The new recommended way to manage cluster applications is to use the [cluster management project template](management_project_template.md). - -- If you want to migrate your GitLab managed apps management to this template, read - [migrating from GitLab managed apps to project template](migrating_from_gma_to_project_template.md). -- If you don't want to use the template, you can also manually manage your applications. - For that, follow the process to - [take ownership of your GitLab Managed Apps](#take-ownership-of-your-gitlab-managed-apps). - -If you are not yet on GitLab 14.0 or later, you can refer to [an older version of this document](https://docs.gitlab.com/13.12/ee/user/clusters/applications.html#install-with-one-click-deprecated). - -## Browse applications logs - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36769) in GitLab 13.2. - -Logs produced by pods running **GitLab Managed Apps** can be browsed using -[**Log Explorer**](../project/clusters/kubernetes_pod_logs.md). - -## Take ownership of your GitLab Managed Apps - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327803) in GitLab 13.12. - -With the removal of the One-click install method in GitLab 14.0, -the **Applications** tab (under your project's **Infrastructure > Kubernetes clusters**) -is no longer displayed: - - - -This tab was dedicated to installing and maintaining GitLab Managed Apps. -To continue managing your installed applications, one of the possible ways is to -install [Helm](https://helm.sh/) locally, as described below. - -### View installed applications - -To view the applications you have installed in your cluster through GitLab Managed Apps, -you need to verify the resources you have in the `gitlab-managed-apps` namespace. -On your computer, [configure `kubectl`](https://kubernetes.io/docs/reference/kubectl/overview/) -to connect to your cluster, open the terminal and run: - -```shell -kubectl get all -n gitlab-managed-apps -``` - -If there is no output or the namespace does not exist, you do not have any applications -installed through GitLab Managed Apps. If this is the case, you have nothing else to do. - -### Identify the Helm version - -Next, verify which Helm version GitLab used to install your applications. - -#### For apps installed with Helm v3 - -To list your apps installed with Helm v3, run: - -```shell -kubectl get secrets -n gitlab-managed-apps | grep 'helm.sh/release' -``` - -You can manage these applications with Helm v3 and you don't need any further steps. - -All applications not listed with the command above were installed with Helm v2. - -#### For apps installed with Helm v2 - -If you have apps installed with Helm v2, you can either: - -- A. Install Helm v3 and [upgrade your apps to Helm v3](https://helm.sh/docs/topics/v2_v3_migration/). -- B. Install Helm v2 and keep using this Helm version, which is not recommended as Helm v2 was deprecated in favor of -Helm v3. - -If you choose to keep using Helm v2 (B), follow the steps below to manage your apps: - -1. Install [Helm v2](https://v2.helm.sh/docs/install/) in your computer. -1. Start a local Tiller server: - - ```shell - export TILLER_NAMESPACE=gitlab-managed-apps - tiller -listen localhost:44134 - ``` - -1. In another tab, initialize your Helm client: - - ```shell - export HELM_HOST="localhost:44134" - helm init --client-only - ``` - -1. Now your environment is ready to manage your apps with Helm v2. For example, to list your releases: - - ```shell - helm ls - ``` - -### Cluster integrations - -Some applications were not only installed in your cluster by GitLab through -Managed Apps but were also directly integrated with GitLab. If you had one of -these applications installed before GitLab 14.0, then a corresponding [cluster -integration](integrations.md) has been automatically enabled: - -- [Prometheus cluster integration](integrations.md#prometheus-cluster-integration) -- [Elastic Stack cluster integration](integrations.md#elastic-stack-cluster-integration) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 13.12. +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) in GitLab 15.0. +Use the [cluster management project template](management_project_template.md) instead. diff --git a/doc/user/clusters/cost_management.md b/doc/user/clusters/cost_management.md index c99eef385ce..47dc9c42797 100644 --- a/doc/user/clusters/cost_management.md +++ b/doc/user/clusters/cost_management.md @@ -8,10 +8,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216737) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.5. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + Cluster cost management provides insights into cluster resource usage. GitLab provides an example [`kubecost-cost-model`](https://gitlab.com/gitlab-examples/kubecost-cost-model/) project that uses the GitLab Prometheus integration and diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index 9e4c672ac45..3f38a473128 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -2,289 +2,12 @@ stage: Configure group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: '../../update/removals.md#managed-cluster-applicationsgitlab-ciyml' --- -# Crossplane configuration (DEPRECATED) **(FREE)** +# Crossplane configuration (removed) **(FREE)** -> [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. - -WARNING: -This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. - -After [installing](applications.md#install-crossplane-using-gitlab-cicd) Crossplane, you must configure it for use. -The process of configuring Crossplane includes: - -1. [Configure RBAC permissions](#configure-rbac-permissions). -1. [Configure Crossplane with a cloud provider](#configure-crossplane-with-a-cloud-provider). -1. [Configure managed service access](#configure-managed-service-access). -1. [Set up Resource classes](#setting-up-resource-classes). -1. Use [Auto DevOps configuration options](#auto-devops-configuration-options). -1. [Connect to the PostgreSQL instance](#connect-to-the-postgresql-instance). - -To allow Crossplane to provision cloud services such as PostgreSQL, the cloud provider -stack must be configured with a user account. For example: - -- A service account for GCP. -- An IAM user for AWS. - -Some important notes: - -- This guide uses GCP as an example, but the processes for AWS and Azure are similar. -- Crossplane requires the Kubernetes cluster to be VPC native with Alias IPs enabled, - so the IP addresses of the pods can be routed within the GCP network. - -First, declare some environment variables with configuration for use in this guide: - -```shell -export PROJECT_ID=crossplane-playground # the GCP project where all resources reside. -export NETWORK_NAME=default # the GCP network where your GKE is provisioned. -export REGION=us-central1 # the GCP region where the GKE cluster is provisioned. -``` - -## Configure RBAC permissions - -For GitLab-managed clusters, role-based access control (RBAC) is configured automatically. - -For non-GitLab managed clusters, ensure that the service account for the token -provided can manage resources in the `database.crossplane.io` API group: - -1. Save the following YAML as `crossplane-database-role.yaml`: - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: crossplane-database-role - labels: - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rules: - - apiGroups: - - database.crossplane.io - resources: - - postgresqlinstances - verbs: - - get - - list - - create - - update - - delete - - patch - - watch - ``` - -1. Apply the cluster role to the cluster: - - ```shell - kubectl apply -f crossplane-database-role.yaml - ``` - -## Configure Crossplane with a cloud provider - -See [Configure Your Cloud Provider Account](https://crossplane.github.io/docs/v1.6/) -to configure the installed cloud provider stack with a user account. - -The Secret, and the Provider resource referencing the Secret, must be -applied to the `gitlab-managed-apps` namespace in the guide. Make sure you change that -while following the process. - -## Configure Managed Service Access - -Next, configure connectivity between the PostgreSQL database and the GKE cluster -by either: - -- Using Crossplane as demonstrated below. -- Directly in the GCP console by - [configuring private services access](https://cloud.google.com/vpc/docs/configure-private-services-access). - -1. Run the following command, which creates a `network.yaml` file, and configures - `GlobalAddress` and connection resources: - - ```plaintext - cat > network.yaml <<EOF - --- - # gitlab-ad-globaladdress defines the IP range that will be allocated - # for cloud services connecting to the instances in the given Network. - - apiVersion: compute.gcp.crossplane.io/v1alpha3 - kind: GlobalAddress - metadata: - name: gitlab-ad-globaladdress - spec: - providerRef: - name: gcp-provider - reclaimPolicy: Delete - name: gitlab-ad-globaladdress - purpose: VPC_PEERING - addressType: INTERNAL - prefixLength: 16 - network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - --- - # gitlab-ad-connection is what allows cloud services to use the allocated - # GlobalAddress for communication. Behind the scenes, it creates a VPC peering - # to the network that those service instances actually live. - - apiVersion: servicenetworking.gcp.crossplane.io/v1alpha3 - kind: Connection - metadata: - name: gitlab-ad-connection - spec: - providerRef: - name: gcp-provider - reclaimPolicy: Delete - parent: services/servicenetworking.googleapis.com - network: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - reservedPeeringRangeRefs: - - name: gitlab-ad-globaladdress - EOF - ``` - -1. Apply the settings specified in the file with the following command: - - ```shell - kubectl apply -f network.yaml - ``` - -1. Verify the creation of the network resources, and that both resources are ready and synced. - - ```shell - kubectl describe connection.servicenetworking.gcp.crossplane.io gitlab-ad-connection - kubectl describe globaladdress.compute.gcp.crossplane.io gitlab-ad-globaladdress - ``` - -## Setting up Resource classes - -Use resource classes to define a configuration for the required managed service. -This example defines the PostgreSQL Resource class: - -1. Run the following command, which define a `gcp-postgres-standard.yaml` resource - class containing a default `CloudSQLInstanceClass` with labels: - - ```plaintext - cat > gcp-postgres-standard.yaml <<EOF - apiVersion: database.gcp.crossplane.io/v1beta1 - kind: CloudSQLInstanceClass - metadata: - name: cloudsqlinstancepostgresql-standard - labels: - gitlab-ad-demo: "true" - specTemplate: - writeConnectionSecretsToNamespace: gitlab-managed-apps - forProvider: - databaseVersion: POSTGRES_11_7 - region: $REGION - settings: - tier: db-custom-1-3840 - dataDiskType: PD_SSD - dataDiskSizeGb: 10 - ipConfiguration: - privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - # this should match the name of the provider created in the above step - providerRef: - name: gcp-provider - reclaimPolicy: Delete - --- - apiVersion: database.gcp.crossplane.io/v1beta1 - kind: CloudSQLInstanceClass - metadata: - name: cloudsqlinstancepostgresql-standard-default - annotations: - resourceclass.crossplane.io/is-default-class: "true" - specTemplate: - writeConnectionSecretsToNamespace: gitlab-managed-apps - forProvider: - databaseVersion: POSTGRES_11_7 - region: $REGION - settings: - tier: db-custom-1-3840 - dataDiskType: PD_SSD - dataDiskSizeGb: 10 - ipConfiguration: - privateNetwork: projects/$PROJECT_ID/global/networks/$NETWORK_NAME - # this should match the name of the provider created in the above step - providerRef: - name: gcp-provider - reclaimPolicy: Delete - EOF - ``` - -1. Apply the resource class configuration with the following command: - - ```shell - kubectl apply -f gcp-postgres-standard.yaml - ``` - -1. Verify creation of the Resource class with the following command: - - ```shell - kubectl get cloudsqlinstanceclasses - ``` - -The Resource Classes allow you to define classes of service for a managed service. -We could create another `CloudSQLInstanceClass` which requests for a larger or a -faster disk. It could also request for a specific version of the database. - -## Auto DevOps Configuration Options - -You can run the Auto DevOps pipeline with either of the following options: - -- Setting the Environment variables `AUTO_DEVOPS_POSTGRES_MANAGED` and - `AUTO_DEVOPS_POSTGRES_MANAGED_CLASS_SELECTOR` to provision PostgreSQL using Crossplane. -- Overriding values for the Helm chart: - - Set `postgres.managed` to `true`, which selects a default resource class. - Mark the resource class with the annotation - `resourceclass.crossplane.io/is-default-class: "true"`. The CloudSQLInstanceClass - `cloudsqlinstancepostgresql-standard-default` is used to satisfy the claim. - - Set `postgres.managed` to `true` with `postgres.managedClassSelector` - providing the resource class to choose, based on labels. In this case, the - value of `postgres.managedClassSelector.matchLabels.gitlab-ad-demo="true"` - selects the CloudSQLInstance class `cloudsqlinstancepostgresql-standard` - to satisfy the claim request. - -The Auto DevOps pipeline should provision a PostgresqlInstance when it runs successfully. - -To verify the PostgreSQL instance was created, run this command. When the `STATUS` -field of the PostgresqlInstance changes to `BOUND`, it's successfully provisioned: - -```shell -$ kubectl get postgresqlinstance - -NAME STATUS CLASS-KIND CLASS-NAME RESOURCE-KIND RESOURCE-NAME AGE -staging-test8 Bound CloudSQLInstanceClass cloudsqlinstancepostgresql-standard CloudSQLInstance xp-ad-demo-24-staging-staging-test8-jj55c 9m -``` - -The endpoint of the PostgreSQL instance, and the user credentials, are present in -a secret called `app-postgres` within the same project namespace. You can verify the -secret with the following command: - -```shell -$ kubectl describe secret app-postgres - -Name: app-postgres -Namespace: xp-ad-demo-24-staging -Labels: <none> -Annotations: crossplane.io/propagate-from-name: 108e460e-06c7-11ea-b907-42010a8000bd - crossplane.io/propagate-from-namespace: gitlab-managed-apps - crossplane.io/propagate-from-uid: 10c79605-06c7-11ea-b907-42010a8000bd - -Type: Opaque - -Data -==== -privateIP: 8 bytes -publicIP: 13 bytes -serverCACertificateCert: 1272 bytes -serverCACertificateCertSerialNumber: 1 bytes -serverCACertificateCreateTime: 24 bytes -serverCACertificateExpirationTime: 24 bytes -username: 8 bytes -endpoint: 8 bytes -password: 27 bytes -serverCACertificateCommonName: 98 bytes -serverCACertificateInstance: 41 bytes -serverCACertificateSha1Fingerprint: 40 bytes -``` - -## Connect to the PostgreSQL instance - -Follow this [GCP guide](https://cloud.google.com/sql/docs/postgres/connect-kubernetes-engine) if you -would like to connect to the newly provisioned PostgreSQL database instance on CloudSQL. +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) +in GitLab 14.5. and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) +in GitLab 15.0. Use [crossplane](http://crossplane.io/) directly instead. diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index 71eb08ee640..4ba0de3bf55 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -9,10 +9,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13392) in GitLab 12.3 for group-level clusters. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14809) in GitLab 12.4 for instance-level clusters. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + Cluster environments provide a consolidated view of which CI [environments](../../ci/environments/index.md) are deployed to the Kubernetes cluster and it: diff --git a/doc/user/clusters/img/applications_tab_v13_12.png b/doc/user/clusters/img/applications_tab_v13_12.png Binary files differdeleted file mode 100644 index c4f1a8862e8..00000000000 --- a/doc/user/clusters/img/applications_tab_v13_12.png +++ /dev/null diff --git a/doc/user/clusters/integrations.md b/doc/user/clusters/integrations.md index 74f6ec283ea..a6dbb5fe0d7 100644 --- a/doc/user/clusters/integrations.md +++ b/doc/user/clusters/integrations.md @@ -6,16 +6,20 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Cluster integrations (DEPRECATED) **(FREE)** -> [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + GitLab provides several ways to integrate applications to your Kubernetes cluster. To enable cluster integrations, first add a Kubernetes cluster to a GitLab -[project](../project/clusters/add_remove_clusters.md) or +[project](../project/clusters/index.md) or [group](../group/clusters/index.md) or [instance](../instance/clusters/index.md). @@ -97,9 +101,21 @@ To enable the Prometheus integration for your cluster: 1. Click **Save changes**. 1. Go to the **Health** tab to see your cluster's metrics. -## Elastic Stack cluster integration +## Elastic Stack cluster integration **(FREE SELF)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61077) in GitLab 13.12. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 14.7. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/360182) behind a [feature flag](../../administration/feature_flags.md) named `monitor_logging` in GitLab 15.0. Disabled by default. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61077) in GitLab 13.12. +WARNING: +This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) +in GitLab 14.7. +It will be removed completely in GitLab 15.2. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `monitor_logging`. +On GitLab.com, this feature is not available. +This feature is not recommended for production use. You can integrate your cluster with [Elastic Stack](https://www.elastic.co/elastic-stack/) to index and [query your pod diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index 7658bc41dd9..d59edb1e2c2 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -8,12 +8,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32810) in GitLab 12.5. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: The cluster management project was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. To manage cluster applications, use the [GitLab agent](agent/index.md) with the [Cluster Management Project Template](management_project_template.md). +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + A project can be designated as the management project for a cluster. A management project can be used to run deployment jobs with Kubernetes diff --git a/doc/user/clusters/management_project_template.md b/doc/user/clusters/management_project_template.md index ab17e462c6a..8ca1bf5d57f 100644 --- a/doc/user/clusters/management_project_template.md +++ b/doc/user/clusters/management_project_template.md @@ -32,7 +32,7 @@ If you have already configured the agent and connected a cluster with GitLab: 1. [Create a project from the cluster management project template](#create-a-project-based-on-the-cluster-management-project-template). 1. In the project where you configured your agent, - [grant the agent access to the new project](agent/ci_cd_tunnel.md#authorize-the-agent). + [grant the agent access to the new project](agent/ci_cd_workflow.md#authorize-the-agent). 1. In the new project, create an [environment variable](../../ci/variables/index.md#add-a-cicd-variable-to-a-project) named `$KUBE_CONTEXT` and set the value to `path/to/agent-configuration-project:your-agent-name`. @@ -78,7 +78,7 @@ This image contains a set of Bash utility scripts to support [Helm v3 releases]( The template contains a [Helmfile](https://github.com/roboll/helmfile) you can use to manage cluster applications with [Helm v3](https://helm.sh/). -This file has a list of paths to other Helmfiles for each app. They're all commented out by default, so you must uncomment +This file has a list of paths to other Helm files for each app. They're all commented out by default, so you must uncomment the paths for the apps that you would like to use in your cluster. By default, each `helmfile.yaml` in these sub-paths has the attribute `installed: true`. This means that every time @@ -93,12 +93,7 @@ application in the template. The [built-in supported applications](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/tree/master/applications) are: -- [Apparmor](../infrastructure/clusters/manage/management_project_applications/apparmor.md) - [Cert-manager](../infrastructure/clusters/manage/management_project_applications/certmanager.md) -- [Cilium](../infrastructure/clusters/manage/management_project_applications/cilium.md) -- [Elastic Stack](../infrastructure/clusters/manage/management_project_applications/elasticstack.md) -- [Falco](../infrastructure/clusters/manage/management_project_applications/falco.md) -- [Fluentd](../infrastructure/clusters/manage/management_project_applications/fluentd.md) - [GitLab Runner](../infrastructure/clusters/manage/management_project_applications/runner.md) - [Ingress](../infrastructure/clusters/manage/management_project_applications/ingress.md) - [Prometheus](../infrastructure/clusters/manage/management_project_applications/prometheus.md) diff --git a/doc/user/clusters/migrating_from_gma_to_project_template.md b/doc/user/clusters/migrating_from_gma_to_project_template.md index 09453262fbb..9a59d135fa0 100644 --- a/doc/user/clusters/migrating_from_gma_to_project_template.md +++ b/doc/user/clusters/migrating_from_gma_to_project_template.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Migrate from GitLab Managed Apps to Cluster Management Projects **(FREE)** -The [GitLab Managed Apps](applications.md) were deprecated in GitLab 14.0 -in favor of [Cluster Management Projects](management_project.md). +The GitLab Managed Apps were deprecated in GitLab 14.0 +in favor of user-controlled Cluster Management projects. Managing your cluster applications through a project enables you a lot more flexibility to manage your cluster than through the late GitLab Managed Apps. To migrate to the cluster management project you need @@ -21,8 +21,10 @@ follow the steps below. See also [video walk-throughs](#video-walk-throughs) with examples. 1. Create a new project based on the [Cluster Management Project template](management_project_template.md#create-a-project-based-on-the-cluster-management-project-template). -1. [Associate your new Cluster Management Project with your cluster](management_project.md#associate-the-cluster-management-project-with-the-cluster). +1. [Install an agent](agent/install/index.md) for this project in your cluster. +1. Set the `KUBE_CONTEXT` CI/CD variable to the newly installed agent's context, as instructed in the `.gitlab-ci.yml` from the Project Template. 1. Detect apps deployed through Helm v2 releases by using the pre-configured [`.gitlab-ci.yml`](management_project_template.md#the-gitlab-ciyml-file) file: + - In case you had overwritten the default GitLab Managed Apps namespace, edit `.gitlab-ci.yml`, and make sure the script is receiving the correct namespace as an argument: @@ -92,6 +94,7 @@ See also [video walk-throughs](#video-walk-throughs) with examples. chart version proposed in `applications/vault/values.yaml`. - Cert-manager: + - For users on Kubernetes version 1.20 or above, the deprecated cert-manager v0.10 is no longer valid and the upgrade includes a breaking change. So we suggest that you [backup and uninstall cert-manager v0.10](#backup-and-uninstall-cert-manager-v010), and install the latest cert-manager instead. To install this version, uncomment `applications/cert-manager/helmfile.yaml` diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index a2172b72572..659c0326728 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -134,7 +134,7 @@ License Compliance can be configured using CI/CD variables. | `ASDF_PYTHON_VERSION` | no | Version of Python to use for the scan. [Configuration](#selecting-the-version-of-python) | | `ASDF_RUBY_VERSION` | no | Version of Ruby to use for the scan. | | `GRADLE_CLI_OPTS` | no | Additional arguments for the Gradle executable. If not supplied, defaults to `--exclude-task=test`. | -| `LICENSE_FINDER_CLI_OPTS` | no | Additional arguments for the `license_finder` executable. For example, if you have multiple projects in nested directories, you can update your `.gitlab-ci-yml` template to specify a recursive scan, like `LICENSE_FINDER_CLI_OPTS: '--recursive'`. | +| `LICENSE_FINDER_CLI_OPTS` | no | Additional arguments for the `license_finder` executable. For example, if you have multiple projects in nested directories, you can update your `.gitlab-ci.yml` template to specify a recursive scan, like `LICENSE_FINDER_CLI_OPTS: '--recursive'`. | | `LM_JAVA_VERSION` | no | Version of Java. If set to `11`, Maven and Gradle use Java 11 instead of Java 8. [Configuration](#selecting-the-version-of-java) | | `LM_PYTHON_VERSION` | no | Version of Python. If set to `3`, dependencies are installed using Python 3 instead of Python 2.7. [Configuration](#selecting-the-version-of-python) | | `MAVEN_CLI_OPTS` | no | Additional arguments for the `mvn` executable. If not supplied, defaults to `-DskipTests`. | @@ -506,7 +506,7 @@ example: } ``` -If credentials are required to authenticate then you can configure a [protected CI/CD variable](../../../ci/variables/index.md#protect-a-cicd-variable) +If credentials are required to authenticate then you can configure a [protected CI/CD variable](../../../ci/variables/index.md#protected-cicd-variables) following the naming convention described in the [`CONAN_LOGIN_USERNAME` documentation](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name). #### Custom root certificates for Conan @@ -853,7 +853,7 @@ A full list of variables can be found in [CI/CD variables](#available-cicd-varia To find out what tools are pre-installed in the `license_scanning` Docker image use the following command: ```shell -$ docker run --entrypoint='' registry.gitlab.com/security-products/license-finder:3 /bin/bash -lc 'asdf list' +$ docker run --entrypoint='' registry.gitlab.com/security-products/license-finder:4 /bin/bash -lc 'asdf list' golang 1.14 gradle @@ -880,7 +880,7 @@ sbt To interact with the `license_scanning` runtime environment use the following command: ```shell -$ docker run -it --entrypoint='' registry.gitlab.com/security-products/license-finder:3 /bin/bash -l +$ docker run -it --entrypoint='' registry.gitlab.com/security-products/license-finder:4 /bin/bash -l root@6abb70e9f193:~# ``` diff --git a/doc/user/crm/index.md b/doc/user/crm/index.md index 7fc11add2cd..7a400205e30 100644 --- a/doc/user/crm/index.md +++ b/doc/user/crm/index.md @@ -6,12 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Customer relations management (CRM) **(FREE)** -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `customer_relations`. -On GitLab.com, this feature is not available. - > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2256) in GitLab 14.6 [with a flag](../../administration/feature_flags.md) named `customer_relations`. Disabled by default. > - In GitLab 14.8 and later, you can [create contacts and organizations only in root groups](https://gitlab.com/gitlab-org/gitlab/-/issues/350634). +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/346082) in GitLab 15.0. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `customer_relations`. +On GitLab.com, this feature is available. With customer relations management (CRM) you can create a record of contacts (individuals) and organizations (companies) and relate them to issues. @@ -118,7 +119,7 @@ organizations using the GraphQL API. ### View issues linked to a contact -To view a contact's issues: +To view a contact's issues, select a contact from the issue sidebar, or: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Customer relations > Contacts**. @@ -166,11 +167,12 @@ API. ## Autocomplete contacts **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2256) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `contacts_autocomplete`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2256) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `contacts_autocomplete`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/352123) in GitLab 15.0. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `contacts_autocomplete`. -On GitLab.com, this feature is not available. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `contacts_autocomplete`. +On GitLab.com, this feature is available. This feature is not ready for production use. When you use the `/add_contacts` or `/remove_contacts` quick actions, follow them with `[contact:` and an autocomplete list appears: diff --git a/doc/user/discussions/img/add_internal_note_v15_0.png b/doc/user/discussions/img/add_internal_note_v15_0.png Binary files differnew file mode 100644 index 00000000000..cf052edd5e7 --- /dev/null +++ b/doc/user/discussions/img/add_internal_note_v15_0.png diff --git a/doc/user/discussions/img/confidential_comments_v13_9.png b/doc/user/discussions/img/confidential_comments_v13_9.png Binary files differdeleted file mode 100644 index b5be5a622a9..00000000000 --- a/doc/user/discussions/img/confidential_comments_v13_9.png +++ /dev/null diff --git a/doc/user/discussions/img/create-new-issue_v14_3.png b/doc/user/discussions/img/create-new-issue_v14_3.png Binary files differdeleted file mode 100644 index d76fed6cb42..00000000000 --- a/doc/user/discussions/img/create-new-issue_v14_3.png +++ /dev/null diff --git a/doc/user/discussions/img/create-new-issue_v15.png b/doc/user/discussions/img/create-new-issue_v15.png Binary files differnew file mode 100644 index 00000000000..779196b6ba4 --- /dev/null +++ b/doc/user/discussions/img/create-new-issue_v15.png diff --git a/doc/user/discussions/img/unresolved_threads_v14_1.png b/doc/user/discussions/img/unresolved_threads_v14_1.png Binary files differdeleted file mode 100644 index 806dfdc995c..00000000000 --- a/doc/user/discussions/img/unresolved_threads_v14_1.png +++ /dev/null diff --git a/doc/user/discussions/img/unresolved_threads_v15.png b/doc/user/discussions/img/unresolved_threads_v15.png Binary files differnew file mode 100644 index 00000000000..113af20effc --- /dev/null +++ b/doc/user/discussions/img/unresolved_threads_v15.png diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 8537856ef25..97c3d5f1058 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -150,42 +150,45 @@ Notes are added to the page details. If an issue or merge request is locked and closed, you cannot reopen it. -## Mark a comment as confidential **(FREE SELF)** +## Add an internal note > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207473) in GitLab 13.9 [with a flag](../../administration/feature_flags.md) named `confidential_notes`. Disabled by default. > - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/351143) in GitLab 14.10: you can only mark comments in issues and epics as confidential. Previously, it was also possible for comments in merge requests and snippets. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87403) from "confidential comments" to "internal notes" in GitLab 15.0. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87383) in GitLab 15.0. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `confidential_notes`. -On GitLab.com, this feature is not available. -You should not use this feature for production environments. +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `confidential_notes`. +On GitLab.com, this feature is available. -You can make a comment **in an issue or an epic** confidential, so that it is visible only to you (the commenting user) and -the project members who have at least the Reporter role. +You can add an internal note **to an issue or an epic**. It's then visible only to the following people: + +- Project members who have at least the Reporter role +- Issue or epic author +- Users assigned to the issue or epic Keep in mind: -- You can only mark comments as confidential when you create them. -- You can't change the confidentiality of existing comments. -- Replies to comments use same confidentiality as the original comment. +- Replies to internal notes are also internal. +- You can not turn an internal note into a regular comment. Prerequisites: - You must either: - Have at least the Reporter role for the project. - - Be the issue assignee. - - Be the issue author. + - Be the issue or epic assignee. + - Be the issue or epic author. -To mark a comment as confidential: +To add an internal note: 1. Start adding a new comment. -1. Below the comment, select the **Make this comment confidential** checkbox. -1. Select **Comment**. +1. Below the comment, select the **Make this an internal note** checkbox. +1. Select **Add internal note**. - + -You can also make an [entire issue confidential](../project/issues/confidential_issues.md). +You can also mark an [issue as confidential](../project/issues/confidential_issues.md). ## Show only comments @@ -300,7 +303,7 @@ To resolve a thread: At the top of the page, the number of unresolved threads is updated: - + ### Move all unresolved threads in a merge request to an issue @@ -308,7 +311,7 @@ If you have multiple unresolved threads in a merge request, you can create an issue to resolve them separately. In the merge request, at the top of the page, select **Create issue to resolve all threads** (**{issue-new}**): - + All threads are marked as resolved, and a link is added from the merge request to the newly created issue. @@ -327,12 +330,13 @@ the newly created issue. ### Prevent merge unless all threads are resolved You can prevent merge requests from being merged until all threads are -resolved. +resolved. When this setting is enabled, the **Unresolved threads** counter in a merge request +is shown in orange when at least one thread remains unresolved. 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > General**. 1. Expand **Merge requests**. -1. Under **Merge checks**, select the **All discussions must be resolved** checkbox. +1. Under **Merge checks**, select the **All threads must be resolved** checkbox. 1. Select **Save changes**. ### Automatically resolve threads in a merge request when they become outdated @@ -344,7 +348,7 @@ with a new push. 1. On the left sidebar, select **Settings > General**. 1. Expand **Merge requests**. 1. Under **Merge options**, select the - **Automatically resolve merge request diff discussions when they become outdated** checkbox. + **Automatically resolve merge request diff threads when they become outdated** checkbox. 1. Select **Save changes**. Threads are now resolved if a push makes a diff section outdated. diff --git a/doc/user/free_user_limit.md b/doc/user/free_user_limit.md new file mode 100644 index 00000000000..2340ef254f6 --- /dev/null +++ b/doc/user/free_user_limit.md @@ -0,0 +1,60 @@ +--- +stage: Growth +group: Conversion +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Free user limit **(FREE SAAS)** + +In GitLab 15.1 (June 22, 2022) and later, namespaces in GitLab.com on the Free tier +will be limited to five (5) members per [namespace](group/index.md#namespaces). +This limit applies to top-level groups and personal namespaces. + +In a personal namespace, the limit applies across all projects in your personal +namespace. + +On the transition date, if your namespace has six or more unique members: + +- Five members will keep a status of `Active`. +- Remaining members will get a status of `Over limit` and lose access to the + group. +- Members invited through a group or project invitation outside of the namespace + will be removed. You can add these members back by inviting them through their + username or email address on the **Members** page for your group or project. + +## How active members are determined + +On the transition date, we'll automatically select the members who keep their `Active` status +in the following order, until we reach a total of five: + +1. Members with the Owner or Maintainer role. +1. The most recently active members. + +## Manage members in your namespace + +To help manage your free user limit, +you can view and manage the total number of members across all projects and groups +in your namespace. + +Prerequisite: + +- You must have the Owner role for the group. + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Usage Quotas**. +1. To view all members, select the **Seats** tab. +1. To remove a member, select **Remove user**. + +NOTE: +The **Usage Quotas** page is not available for personal namespaces. You can +view and [remove members](project/members/index.md#remove-a-member-from-a-project) +in each project instead. The five user limit includes all +unique members across all projects in your personal namespace. + +If you need more time to manage your members, or to try GitLab features +with a team of more than five members, you can [start a trial](https://about.gitlab.com/free-trial/). +A trial lasts for 30 days and includes an unlimited number of members. + +## Related topics + +- [GitLab SaaS Free tier frequently asked questions](https://about.gitlab.com/pricing/faq-efficient-free-tier/) diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 6bb45575fc3..e88777a7223 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -9,6 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w This page contains information about the settings that are used on GitLab.com, available to [GitLab SaaS](https://about.gitlab.com/pricing/) customers. +See some of these settings on the [instance configuration page](https://gitlab.com/help/instance_configuration) of GitLab.com. + ## Password requirements GitLab.com has the following requirements for passwords on new accounts and password changes: @@ -97,6 +99,10 @@ If you are on: - Premium tier and above, you can disable this by changing the [group setting](../group/index.md#enable-delayed-project-deletion). - Free tier, you cannot disable this setting or restore projects. +## Inactive project deletion + +[Inactive project deletion](../../administration/inactive_project_deletion.md) is disabled on GitLab.com. + ## Alternative SSH port GitLab.com can be reached by using a [different SSH port](https://about.gitlab.com/blog/2016/02/18/gitlab-dot-com-now-supports-an-alternate-git-plus-ssh-port/) for `git+ssh`. @@ -127,9 +133,9 @@ Below are the settings for [GitLab Pages](https://about.gitlab.com/stages-devops | IP address | `35.185.44.232` | - | | Custom domains support | **{check-circle}** Yes | **{dotted-circle}** No | | TLS certificates support | **{check-circle}** Yes | **{dotted-circle}** No | -| Maximum size (compressed) | 1 GB | 100 MB | +| [Maximum size](../../administration/pages/index.md#set-global-maximum-pages-size-per-project) (compressed) | 1 GB | 100 MB | -The maximum size of your Pages site is regulated by the artifacts maximum size, +The maximum size of your Pages site is also regulated by the artifacts maximum size, which is part of [GitLab CI/CD](#gitlab-cicd). There are also [rate limits set for GitLab Pages](#gitlabcom-specific-rate-limits). @@ -330,7 +336,7 @@ after the limits change in January, 2021: | **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | | **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | | **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | -| **Issue creation** | **300** requests per minute | **300** requests per minute | +| **Issue creation** | **300** requests per minute | **200** requests per minute | | **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | | **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | | **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | diff --git a/doc/user/group/epics/linked_epics.md b/doc/user/group/epics/linked_epics.md index b695bae39e4..89699661efa 100644 --- a/doc/user/group/epics/linked_epics.md +++ b/doc/user/group/epics/linked_epics.md @@ -6,12 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Linked epics **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353473) in GitLab 14.9 [with a flag](../../../administration/feature_flags.md) named `related_epics_widget`. Enabled by default. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, -ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) -named `related_epics_widget`. On GitLab.com, this feature is available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353473) in GitLab 14.9 [with a flag](../../../administration/feature_flags.md) named `related_epics_widget`. Enabled by default. +> - [Feature flag `related_epics_widget`](https://gitlab.com/gitlab-org/gitlab/-/issues/357089) removed in GitLab 15.0. Linked epics are a bi-directional relationship between any two epics and appear in a block below the epic description. You can link epics in different groups. @@ -38,7 +34,11 @@ To link one epic to another: - **relates to** - **[blocks](#blocking-epics)** - **[is blocked by](#blocking-epics)** -1. Enter the epic number or paste in the full URL of the epic. +1. To enter the linked epic, either: + + - Enter `&`, followed by the epic's number. For example, `&123`. + - Enter `&`, followed by a word from the epic's title. For example, `&Deliver`. + - Paste in the epic's full URL.  diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md index 0185cb9cfdf..6967815bf22 100644 --- a/doc/user/group/import/index.md +++ b/doc/user/group/import/index.md @@ -132,6 +132,8 @@ migrated: - image URL - Boards - Board Lists +- Releases + - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.0). ## Troubleshooting Group Migration diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 085cd054c14..87146329031 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -53,7 +53,7 @@ For example, consider a user named Alex: | GitLab URL | Namespace | | ---------- | --------- | | Alex creates an account with the username `alex`: `https://gitlab.example.com/alex`. | The namespace in this case is `alex`. | -| Alex creates a group for their team with the group name `alex-team`. The group and its projects are available at: `https://gitlab.example.com/alex-team`. | The namespace in this cases is `alex-team`. | +| Alex creates a group for their team with the group name `alex-team`. The group and its projects are available at: `https://gitlab.example.com/alex-team`. | The namespace in this case is `alex-team`. | | Alex creates a subgroup of `alex-team` with the subgroup name `marketing`. The subgroup and its projects are available at: `https://gitlab.example.com/alex-team/marketing`. | The namespace in this case is `alex-team/marketing`. | ## Create a group @@ -279,7 +279,7 @@ To view the activity feed in Atom format, select the Similar to how you [share a project with a group](../project/members/share_project_with_groups.md), you can share a group with another group. To invite a group, you must be a member of it. Members get direct access -to the shared group. This includes members who inherited group membership from a parent group. +to the shared group. This includes members who inherited group membership from a parent group. To share a given group, for example, `Frontend` with another group, for example, `Engineering`: @@ -456,25 +456,28 @@ To restore a group that is marked for deletion: ## Prevent group sharing outside the group hierarchy -This setting is only available on top-level groups. It affects all subgroups. +You can configure a top-level group so its subgroups and projects +cannot invite other groups outside of the top-level group's hierarchy. +This option is only available for top-level groups. -When checked, any group in the top-level group hierarchy can be shared only with other groups in the hierarchy. +For example, in the following group and project hierarchy: -For example, with these groups: - -- **Animals > Dogs** +- **Animals > Dogs > Dog Project** - **Animals > Cats** - **Plants > Trees** -If you select this setting in the **Animals** group: +If you prevent group sharing outside the hierarchy for the **Animals** group: -- **Dogs** can be shared with **Cats**. -- **Dogs** cannot be shared with **Trees**. +- **Dogs** can invite the group **Cats**. +- **Dogs** cannot invite the group **Trees**. +- **Dog Project** can invite the group **Cats**. +- **Dog Project** cannot invite the group **Trees**. To prevent sharing outside of the group's hierarchy: -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Permissions and group features**. 1. Select **Prevent members from sending invitations to groups outside of `<group_name>` and its subgroups**. 1. Select **Save changes**. @@ -610,15 +613,16 @@ applies to: You should consider these security implications before configuring IP address restrictions: -- **SSH requests**: While you can restrict HTTP traffic on GitLab.com with IP address restrictions, +- **SSH requests, including `git` operations will fail from all IP addresses**: While you can restrict HTTP traffic on GitLab.com with IP address restrictions, they cause SSH requests, including Git operations over SSH, to fail. For more information, read [issue 271673](https://gitlab.com/gitlab-org/gitlab/-/issues/271673). -- **Administrators and group owners**: Users with these permission levels can always +- **Administrators and group owners can access group settings from any IP address**: Users with these permission levels can always access the group settings, regardless of IP restriction, but they cannot access projects belonging to the group when accessing from a disallowed IP address. -- **GitLab API and runner activities**: Only the [group](../../api/groups.md) (including all +- **Some GitLab API endpoints will remain accessible from any IP**: Only the [group](../../api/groups.md) (including all [group resources](../../api/api_resources.md#group-resources)) APIs and [project](../../api/api_resources.md#project-resources) (including all [project resources](../../api/api_resources.md#project-resources)) APIs are protected by IP address restrictions. +- **Activities performed by GitLab Runners are not bound by IP restrictions**: When you register a runner, it is not bound by the IP restrictions. When the runner requests a new job or an update to a job's state, it is also not bound by the IP restrictions. But when the running CI/CD job sends Git requests from a diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 1c316f2157d..858b13b87b8 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -41,7 +41,8 @@ From there you can create a new iteration or select an iteration to get a more d ## Create an iteration -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) in GitLab 14.10. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) in GitLab 14.10. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. WARNING: Manual iteration management is in its end-of-life process. Creating an iteration is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) @@ -49,7 +50,7 @@ in GitLab 14.10, and is planned for removal in GitLab 16.0. Prerequisites: -- You must have at least the Developer role for a group. +- You must have at least the Reporter role for a group. To create an iteration: @@ -63,6 +64,7 @@ To create an iteration: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218277) in GitLab 13.2. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) in GitLab 14.10. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. WARNING: Editing all attributes, with the exception of `description` is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) @@ -71,7 +73,7 @@ In the future only editing an iteration's `description` will be allowed. Prerequisites: -- You must have at least the Developer role for a group. +- You must have at least the Reporter role for a group. To edit an iteration, select the three-dot menu (**{ellipsis_v}**) > **Edit**. @@ -79,6 +81,7 @@ To edit an iteration, select the three-dot menu (**{ellipsis_v}**) > **Edit**. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292268) in GitLab 14.3. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) in GitLab 14.10. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. WARNING: Manual iteration management is in its end-of-life process. Deleting an iteration is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) @@ -86,7 +89,7 @@ in GitLab 14.10, and is planned for removal in GitLab 16.0. Prerequisites: -- You must have at least the Developer role for a group. +- You must have at least the Reporter role for a group. To delete an iteration, select the three-dot menu (**{ellipsis_v}**) > **Delete**. @@ -169,37 +172,65 @@ To group issues by label: ## Iteration cadences -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5077) in GitLab 14.1. -> - Deployed behind a [feature flag](../../feature_flags.md), named `iteration_cadences`, disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an -administrator to [enable the feature flag](../../../administration/feature_flags.md) named -`iteration_cadences` for a root group. -On GitLab.com, this feature is not available. This feature is not ready for production use. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5077) in GitLab 14.1 [with a flag](../../../administration/feature_flags.md), named `iteration_cadences`. Disabled by default. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/354977) in GitLab 15.0: All scheduled iterations must start on the same day of the week as the cadence start day. Start date of cadence cannot be edited after the first iteration starts. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/354878) in GitLab 15.0. Iteration cadences automate iteration scheduling. You can use them to -automate creating iterations every 1, 2, 3, 4, or 6 weeks. You can also +automate creating iterations every 1, 2, 3, or 4 weeks. You can also configure iteration cadences to automatically roll over incomplete issues to the next iteration. ### Create an iteration cadence +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. + Prerequisites: -- You must have at least the Developer role for a group. +- You must have at least the Reporter role for a group. To create an iteration cadence: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Issues > Iterations**. 1. Select **New iteration cadence**. -1. Fill out required fields, and select **Create iteration cadence**. The cadence list page opens. +1. Complete the fields. + - Enter the title and description of the iteration cadence. + - Enter the first iteration start date of the iteration cadence. Iterations will be scheduled to + begin on the same day of the week as the day of the week of the start date. + - From the **Duration** dropdown list, select how many weeks each iteration should last. + - From the **Upcoming iterations** dropdown list, select how many upcoming iterations should be + created and maintained by GitLab. + - Optional. To move incomplete issues to the next iteration, select **Roll over issues**. +1. Select **Create cadence**. The cadence list page opens. + +### Edit an iteration cadence + +Prerequisites: + +- You must have at least the Developer role for a group. + +To edit an iteration cadence: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Issues > Iterations**. +1. Select **Edit iteration cadence**. + +When you edit the **Duration**, **Upcoming iterations**, or **First iteration start date** fields, +only upcoming iterations are affected. + +You can edit the first iteration start date of a cadence if the cadence has not started yet. + +Editing **Upcoming iterations** is a non-destructive action. +If ten upcoming iterations already exist, changing the number under **Upcoming iterations** to `2` +doesn't delete the eight existing upcoming iterations. ### Delete an iteration cadence +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. + Prerequisites: -- You must have at least the Developer role for a group. +- You must have at least the Reporter role for a group. Deleting an iteration cadence also deletes all iterations within that cadence. @@ -210,18 +241,67 @@ To delete an iteration cadence: 1. Select the three-dot menu (**{ellipsis_v}**) > **Delete cadence** for the cadence you want to delete. 1. Select **Delete cadence** in the confirmation modal. -### Convert manual cadence to use automatic scheduling +### Manual iteration cadences + +When you **enable** the iteration cadences feature, all previously +created iterations are added to a default iteration cadence. +You can continue to add, edit, and remove iterations in +this default cadence. + +#### Convert a manual cadence to use automatic scheduling WARNING: -The upgrade is irreversible. After it's done, manual iteration cadences cannot be created. +The upgrade is irreversible. After it's done, a new manual iteration cadence cannot be created. -When you **enable** the iteration cadences feature, all iterations are added -to a default iteration cadence. -In this default iteration cadence, you can continue to add, edit, and remove iterations. +Prerequisites: +- You must have created [iterations](#iterations) without cadences before enabling iteration cadences for your group. To upgrade the iteration cadence to use the automation features: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Issues > Iterations**. 1. Select the three-dot menu (**{ellipsis_v}**) > **Edit cadence** for the cadence you want to upgrade. -1. Fill out required fields, and select **Save changes**. +1. Complete the required fields **Duration** and **Upcoming iterations**. +1. Select **Save changes**. + +#### Start dates of converted cadences + +The first iteration start date of your converted cadence is set to the start date of its +**first** existing iteration. + +If you attempt to set a new start date, the conversion fails with an error message. +If your manual cadence is empty, converting it to use automatic scheduling is effectively +the same as creating a new automated cadence. + +GitLab will start scheduling new iterations on the same day of the week as the start date, +starting from the nearest such day from the current date. + +During the conversion process GitLab does not delete or modify existing **ongoing** or +**closed** iterations. If you have iterations with start dates in the future, +they are updated to fit your cadence settings. + +#### Converted cadences example + +For example, suppose it's Friday, April 15, and you have three iterations in a manual cadence: + +- Monday, April 4 - Friday, April 8 (closed) +- Tuesday, April 12 - Friday, April 15 (ongoing) +- Tuesday, May 3 - Friday, May 6 (upcoming) + +On Friday, April 15, you convert the manual cadence +to automate scheduling iterations every week, up to two upcoming iterations. + +The first iteration is closed, and the second iteration is ongoing, +so they aren't deleted or modified in the conversion process. + +To observe the weekly duration, the third iteration is updated so that it: + +- Starts on Monday, April 18 - which is the nearest date that is Monday. +- Ends on Sunday, April 24. + +Finally, to always have two upcoming iterations, an additional iteration is scheduled: + +- Monday, April 4 - Friday, April 8 (closed) +- Tuesday, April 12 - Friday, April 15 (ongoing) +- Monday, April 18 - Sunday, April 24 (upcoming) +- Monday, April 25 - Sunday, May 1 (upcoming) diff --git a/doc/user/group/roadmap/index.md b/doc/user/group/roadmap/index.md index 89c5c6ed466..10ca6d139ad 100644 --- a/doc/user/group/roadmap/index.md +++ b/doc/user/group/roadmap/index.md @@ -1,5 +1,4 @@ --- -type: reference stage: Plan group: Product Planning info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments @@ -7,7 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Roadmap **(PREMIUM)** -> - Introduced in GitLab 10.5. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/198062) from GitLab Ultimate to GitLab Premium in 12.9. > - In [GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/5164) and later, the epic bars show epics' title, progress, and completed weight percentage. > - Milestones appear in roadmaps in [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/6802), and later. @@ -27,10 +25,10 @@ You can expand epics that contain child epics to show their child epics in the r You can select the chevron (**{chevron-down}**) next to the epic title to expand and collapse the child epics. -On top of the milestone bars, you can see their title. -When you hover over a milestone bar or title, a popover appears with its title, start date, and due -date. You can also select the chevron (**{chevron-down}**) next to the **Milestones** heading to -toggle the list of the milestone bars. +On top of the milestone bars, you can see their title. When you point to a +milestone bar or title, a popover appears with its title, start date, and due +date. You can also select the chevron (**{chevron-down}**) next to the **Milestones** +heading to toggle the list of the milestone bars.  @@ -41,8 +39,8 @@ toggle the list of the milestone bars. > - Filtering by epic [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218623) in GitLab 13.11. > - Filtering by milestone [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323917) in GitLab 14.5. -WARNING: -Filtering roadmaps by milestone might not be available to you. Check the **version history** note above for details. +NOTE: +Filtering roadmaps by milestone might not be available to you. Be sure to review this section's version history for details. When you want to explore a roadmap, there are several ways to make it easier by sorting epics or filtering them by what's important for you. @@ -52,8 +50,9 @@ You can sort epics in the Roadmap view by: - Start date - Due date -Each option contains a button that toggles the sort order between **ascending** and **descending**. -The sort option and order persist when browsing Epics, including the [epics list view](../epics/index.md). +Each option contains a button that toggles the sort order between **ascending** +and **descending**. The sort option and order persist when browsing Epics, including +the [epics list view](../epics/index.md). You can also filter epics in the Roadmap view by the epics': @@ -66,7 +65,7 @@ You can also filter epics in the Roadmap view by the epics':  -Roadmaps can also be [visualized inside an epic](../epics/index.md#roadmap-in-epics). +You can also [visualize roadmaps inside of an epic](../epics/index.md#roadmap-in-epics). ### Roadmap settings @@ -78,38 +77,40 @@ When you enable the roadmap settings sidebar, you can use it to refine epics sho You can configure the following: - Select date range. -- Turn milestones on or off and select whether to show all, group, subgroup, or project milestones. +- Turn milestones on or off, and select whether to show all, group, subgroup, or + project milestones. - Show all, open, or closed epics. - Turn progress tracking for child issues on or off and select whether to use issue weights or counts. - The progress tracking setting is not saved in user preferences but is saved or shared using URL parameters. +The progress tracking setting isn't saved in user preferences, but is saved or +shared using URL parameters. ## Timeline duration -> - Introduced in GitLab 11.0. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/198062) from GitLab Ultimate to GitLab Premium in 12.9. +> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/198062) from GitLab Ultimate to GitLab Premium in 12.9. ### Date range presets -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/204994) in GitLab 14.3. [Deployed behind the `roadmap_daterange_filter` flag](../../../administration/feature_flags.md), disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/204994) in GitLab 14.3 [with a flag](../../../administration/feature_flags.md) named `roadmap_daterange_filter`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/323917) in GitLab 14.3. -> - [Feature flag `roadmap_daterange_filter` removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72419) in GitLab 14.5. +> - Generally available in GitLab 14.5. [Feature flag `roadmap_daterange_filter`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72419) removed. -Roadmap provides three date range options, each with predetermined timeline duration: +Roadmap provides these date range options, each with a predetermined timeline duration: -- **This quarter**: includes weeks present in current quarter. -- **This year**: includes weeks or months present in current year. -- **Within 3 years**: includes weeks, months, or quarters present in the previous 18 months and - upcoming 18 months (that is, three years in total). +- **This quarter**: Includes the weeks present in the current quarter. +- **This year**: Includes the weeks or months present in the current year. +- **Within 3 years**: Includes the weeks, months, or quarters present both in + the previous 18 months and the upcoming 18 months (three years in total). ### Layout presets -Depending on selected [date range preset](#date-range-presets), Roadmap supports the following layout presets: +Depending on selected [date range preset](#date-range-presets), Roadmap supports +these layout presets: -- **Quarters**: only available when the "Within 3 years" date range is selected. -- **Months**: available when either "This year" or "Within 3 years" date range is selected. -- **Weeks** (default): available for all the date range presets. +- **Quarters**: Available only when the **Within 3 years** date range is selected. +- **Months**: Available when either **This year** or **Within 3 years** date range is selected. +- **Weeks** (default): Available for all the date range presets. ### Quarters @@ -125,12 +126,11 @@ the timeline header represent the month of the quarter.  -In the **Months** preset, roadmap shows epics and milestones which have start or due dates -**falling within** or -**going through** currently selected date range preset, where **today** -is shown by the vertical red line in the timeline. The sub-headers underneath the month name on -the timeline header represent the date on starting day (Sunday) of the week. This preset is -selected by default. +In the **Months** preset, roadmap shows epics and milestones which have start or +due dates **falling within** or **going through** currently selected date range +preset, where **today** is shown by the vertical red line in the timeline. The +sub-headers underneath the month name on the timeline header represent the date +on the start day (Sunday) of the week. This preset is selected by default. ### Weeks diff --git a/doc/user/group/saml_sso/group_managed_accounts.md b/doc/user/group/saml_sso/group_managed_accounts.md index bffaef40800..6771ff8739a 100644 --- a/doc/user/group/saml_sso/group_managed_accounts.md +++ b/doc/user/group/saml_sso/group_managed_accounts.md @@ -107,14 +107,14 @@ Since personal access tokens are the only token needed for programmatic access t ### Set a limit Only a GitLab administrator or an owner of a group-managed account can set a limit. When this field -is left empty, the [instance-level restriction](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens) +is left empty, the [instance-level restriction](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) on the lifetime of personal access tokens apply. To set a limit on how long personal access tokens are valid for users in a group managed account: 1. Navigate to the **Settings > General** page in your group's sidebar. 1. Expand the **Permissions and group features** section. -1. Fill in the **Maximum allowable lifetime for personal access tokens (days)** field. +1. Fill in the **Maximum allowable lifetime for access tokens (days)** field. 1. Click **Save changes**. Once a lifetime for personal access tokens is set: diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 4d122e337db..1b480a52920 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -129,7 +129,7 @@ SSO has the following effects when enabled: - For Git activity over SSH and HTTPS, users must have at least one active session signed-in through SSO before they can push to or pull from a GitLab repository. - Git activity originating from CI/CD jobs do not have the SSO check enforced. -- Credentials that are not tied to regular users (for example, access tokens and deploy keys) do not have the SSO check enforced. +- Credentials that are not tied to regular users (for example, project and group access tokens, and deploy keys) do not have the SSO check enforced. - Users must be signed-in through SSO before they can pull images using the [Dependency Proxy](../../packages/dependency_proxy/index.md). When SSO is enforced, users are not immediately revoked. If the user: @@ -589,7 +589,7 @@ Here are possible causes and solutions: | Cause | Solution | | ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | You've tried to link multiple SAML identities to the same user, for a given identity provider. | Change the identity that you sign in with. To do so, [unlink the previous SAML identity](#unlinking-accounts) from this GitLab account before attempting to sign in again. | -| The NameID changes everytime the user requests SSO identification | Check the NameID is not set with `Transient` format, or the NameID is not changing on subsequent requests.| +| The NameID changes every time the user requests SSO identification | Check the NameID is not set with `Transient` format, or the NameID is not changing on subsequent requests.| ### Message: "SAML authentication failed: Email has already been taken" diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index 3960c97142e..bc1799c2e54 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -110,7 +110,7 @@ Before you start this section: After the above steps are complete: 1. Sign in to Okta. -1. Ensure you are in the Admin section by selecting the **Admin** button located in the top right. The admin button is not visible from the admin page. +1. Ensure you are in the Admin Area by selecting the **Admin** button located in the top right. The button is not visible from the Admin Area. 1. In the **Application** tab, select **Browse App Catalog**. 1. Search for **GitLab**, find and select on the 'GitLab' application. 1. On the GitLab application overview page, select **Add**. diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index 0666303bcf8..4b791d5a221 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -25,7 +25,7 @@ Group access tokens are similar to [project access tokens](../../project/setting and [personal access tokens](../../profile/personal_access_tokens.md), except they are associated with a group rather than a project or user. -In self-managed instances, group access tokens are subject to the same [maximum lifetime limits](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens) as personal access tokens if the limit is set. +In self-managed instances, group access tokens are subject to the same [maximum lifetime limits](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) as personal access tokens if the limit is set. You can use group access tokens: @@ -50,7 +50,7 @@ To create a group access token: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Settings > Access Tokens**. 1. Enter a name. The token name is visible to any user with permissions to view the group. -1. Optional. Enter an expiry date for the token. The token will expire on that date at midnight UTC. An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. +1. Optional. Enter an expiry date for the token. The token will expire on that date at midnight UTC. An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. 1. Select a role for the token. 1. Select the [desired scopes](#scopes-for-a-group-access-token). 1. Select **Create group access token**. diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md index 7b63466656d..23a638fb98c 100644 --- a/doc/user/group/settings/import_export.md +++ b/doc/user/group/settings/import_export.md @@ -86,7 +86,7 @@ To export the contents of a group: NOTE: The maximum import file size can be set by the Administrator, default is `0` (unlimited). -As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin UI](../../admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. +As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](../../../api/settings.md#change-application-settings) or the [Admin Area](../../admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50MB to 0 in GitLab 13.8. You can also use the [group import/export API](../../../api/group_import_export.md). diff --git a/doc/user/group/value_stream_analytics/img/new_value_stream_v13_12.png b/doc/user/group/value_stream_analytics/img/new_value_stream_v13_12.png Binary files differdeleted file mode 100644 index d64ec31aabf..00000000000 --- a/doc/user/group/value_stream_analytics/img/new_value_stream_v13_12.png +++ /dev/null diff --git a/doc/user/group/value_stream_analytics/img/vsa_aggregated_data_toggle_v14_9.png b/doc/user/group/value_stream_analytics/img/vsa_aggregated_data_toggle_v14_9.png Binary files differdeleted file mode 100644 index 5ad8026b8fd..00000000000 --- a/doc/user/group/value_stream_analytics/img/vsa_aggregated_data_toggle_v14_9.png +++ /dev/null diff --git a/doc/user/group/value_stream_analytics/img/vsa_custom_stage_v13_10.png b/doc/user/group/value_stream_analytics/img/vsa_custom_stage_v13_10.png Binary files differdeleted file mode 100644 index 9345c4023de..00000000000 --- a/doc/user/group/value_stream_analytics/img/vsa_custom_stage_v13_10.png +++ /dev/null diff --git a/doc/user/group/value_stream_analytics/img/vsa_default_stage_v13_10.png b/doc/user/group/value_stream_analytics/img/vsa_default_stage_v13_10.png Binary files differdeleted file mode 100644 index a29689a2c18..00000000000 --- a/doc/user/group/value_stream_analytics/img/vsa_default_stage_v13_10.png +++ /dev/null diff --git a/doc/user/group/value_stream_analytics/index.md b/doc/user/group/value_stream_analytics/index.md index 3fce9baa9ce..4be97779603 100644 --- a/doc/user/group/value_stream_analytics/index.md +++ b/doc/user/group/value_stream_analytics/index.md @@ -11,6 +11,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w Value stream analytics provides metrics about each stage of your software development process. +A **value stream** is the entire work process that delivers value to customers. For example, +the [DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/) is a value stream that starts +with the "manage" stage and ends with the "protect" stage. + Use value stream analytics to identify: - The amount of time it takes to go from an idea to production. @@ -30,7 +34,7 @@ Value stream analytics is also available for [projects](../../analytics/value_st Prerequisite: - You must have at least the Reporter role to view value stream analytics for groups. -- You must create a [custom value stream](#custom-value-streams). Value stream analytics only shows custom value streams created for your group. +- You must create a [custom value stream](#create-a-value-stream-with-gitlab-default-stages). Value stream analytics only shows custom value streams created for your group. To view value stream analytics for your group: @@ -41,9 +45,6 @@ To view value stream analytics for your group: 1. Select the **Filter results** text box. 1. Select a parameter. 1. Select a value or enter text to refine the results. - 1. Select whether to view metrics for items with a start or stop event: - - To view items with a stop event in the date range, turn on the **Filter by stop date** toggle. Enabled by default. - - To view items with a start event in the date range, turn off the **Filter by stop date** toggle. 1. To adjust the date range: - In the **From** field, select a start date. - In the **To** field, select an end date. The charts and list show workflow items created @@ -78,9 +79,6 @@ To view the median time spent in each stage by a group: 1. Select the **Filter results** text box. 1. Select a parameter. 1. Select a value or enter text to refine the results. - 1. Select whether to view metrics for items with a start or stop event: - - To view items with a stop event in the date range, turn on the **Filter by stop date** toggle. Enabled by default. - - To view items with a start event in the date range, turn off the **Filter by stop date** toggle. 1. To adjust the date range: - In the **From** field, select a start date. - In the **To** field, select an end date. @@ -91,7 +89,10 @@ To view the median time spent in each stage by a group: Value stream analytics shows the lead time and cycle time for issues in your groups: - Lead time: Median time from when the issue was created to when it was closed. -- Cycle time: Median time from first commit to issue closed. Commits are associated with issues when users [cross-link them in the commit message](../../project/issues/crosslinking_issues.md#from-commit-messages). +- Cycle time: Median time from first commit to issue closed. GitLab measures cycle time from the earliest +commit of a [linked issue's merge request](../../project/issues/crosslinking_issues.md#from-commit-messages) +to when that issue is closed. The cycle time approach underestimates the lead time because merge request creation +is always later than commit time. To view the lead time and cycle time for issues: @@ -101,9 +102,6 @@ To view the lead time and cycle time for issues: 1. Select the **Filter results** text box. 1. Select a parameter. 1. Select a value or enter text to refine the results. - 1. Select whether to view metrics for items with a start or stop event: - - To view items with a stop event in the date range, turn on the **Filter by stop date** toggle. Enabled by default. - - To view items with a start event in the date range, turn off the **Filter by stop date** toggle. 1. To adjust the date range: - In the **From** field, select a start date. - In the **To** field, select an end date. @@ -124,9 +122,6 @@ To view the lead time for changes for merge requests in your group: 1. Select the **Filter results** text box. 1. Select a parameter. 1. Select a value or enter text to refine the results. - 1. Select whether to view metrics for items with a start or stop event: - - To view items with a stop event in the date range, turn on the **Filter by stop date** toggle. Enabled by default. - - To view items with a start event in the date range, turn off the **Filter by stop date** toggle. 1. To adjust the date range: - In the **From** field, select a start date. - In the **To** field, select an end date. @@ -140,7 +135,7 @@ The **Lead Time for Changes** metrics display below the **Filter results** text To view deployment metrics, you must have a [production environment configured](../../../ci/environments/index.md#deployment-tier-of-environments). -Value stream analytics shows the following deployment metrics for your group: +Value stream analytics shows the following deployment metrics for your group: - Deploys: The number of successful deployments in the date range. - Deployment Frequency: The average number of successful deployments per day in the date range. @@ -153,13 +148,14 @@ To view deployment metrics for your group: 1. Select the **Filter results** text box. 1. Select a parameter. 1. Select a value or enter text to refine the results. - 1. Select whether to view metrics for items with a start or stop event: - - To view items with a stop event in the date range, turn on the **Filter by stop date** toggle. Enabled by default. - - To view items with a start event in the date range, turn off the **Filter by stop date** toggle. 1. To adjust the date range: - In the **From** field, select a start date. - In the **To** field, select an end date. +NOTE: +The date range selector filters items by the event time. This is the time when the currently +selected stage finished for the given item. + The **Deploys** and **Deployment Frequency** metrics display below the **Filter results** text box. Deployment metrics are calculated based on data from the @@ -174,19 +170,22 @@ In GitLab 13.8 and earlier, metrics are calculated based on when the deployment > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335391) in GitLab 14.5 [with a flag](../../../administration/feature_flags.md) named `use_vsa_aggregated_tables`. Disabled by default. > - Filter by stop date toggle [added](https://gitlab.com/gitlab-org/gitlab/-/issues/352428) in GitLab 14.9 > - Data refresh badge [added](https://gitlab.com/gitlab-org/gitlab/-/issues/341739) in GitLab 14.9 +> - Filter by stop date toggle [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84356) in GitLab 14.9 +> - Enable filtering by stop date [added](https://gitlab.com/gitlab-org/gitlab/-/issues/355000) in GitLab 15.0 -Plans for value stream analytics to filter items by stop event instead of start event are tracked in this [epic](https://gitlab.com/groups/gitlab-org/-/epics/6046). With the completion of this work, value stream analytics will only display items with a stop event in the date range. - -To preview this functionality, you can use the **Filter by stop date** toggle to enable or disable this filter until the [default filtering mode is introduced](../../../update/deprecations.md#value-stream-analytics-filtering-calculation-change) and the toggle is removed. +Value stream analytics uses a backend process to collect and aggregate stage-level data, which +ensures it can scale for large groups with a high number of issues and merge requests. Due to this process, +there may be a slight delay between when an action is taken (for example, closing an issue) and when the data +displays on the value stream analytics page. -If you turn on the **Filter by stop date** toggle, the results show items with a stop event within the date range. When this function is enabled, it may take up to 10 minutes for results to show due to data aggregation. There are occasions when it may take longer than 10 minutes for results to display: +It may take up to 10 minutes to process the data and display results. Data collection may take +longer than 10 minutes in the following cases: -- If this is the first time you are viewing value stream analytics and have not yet [created a value stream](#create-a-value-stream). +- If this is the first time you are viewing value stream analytics and have not yet [created a value stream](#create-a-value-stream-with-gitlab-default-stages). - If the group hierarchy has been re-arranged. - If there have been bulk updates on issues and merge requests. -To view when the data was most recently updated, in the right corner next to **Edit**, hover over the **Last updated** badge. This badge is only available if you have turned on the **Filter by start date** toggle. - +To view when the data was most recently updated, in the right corner next to **Edit**, hover over the **Last updated** badge. ## How value stream analytics measures stages @@ -261,80 +260,58 @@ These patterns are not case-sensitive. You can change the name of a project environment in your GitLab CI/CD configuration. -## Custom value streams - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12196) in GitLab 12.9. - -Use custom value streams to create custom stages that align with your own development processes, -and hide default stages. The dashboard depicts stages as a horizontal process -flow. - -### Create a value stream +## Create a value stream with GitLab default stages > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221202) in GitLab 13.3 -A default value stream is readily available for each group. You can create additional value streams -based on the different areas of work that you would like to measure. - -Once created, a new value stream includes the stages that follow -[GitLab workflow](../../../topics/gitlab_flow.md) -best practices. You can customize this flow by adding, hiding or re-ordering stages. - -To create a value stream: +When you create a value stream, you can use GitLab default stages and hide or re-order them to customize. You can also +create custom stages in addition to those provided in the default template. 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. -1. If this is the first time you are creating a value stream, select **Create custom value stream**. Otherwise, in the top right, select the dropdown list and then **Create new Value Stream**. -1. Enter a name for the new Value Stream. - - You can [customize the stages](#create-a-value-stream-with-stages). -1. Select **Create Value Stream**. - - +1. Select **Create new Value Stream**. +1. Enter a name for the value stream. +1. Select **Create from default template**. +1. Customize the default stages: + - To re-order stages, select the up or down arrows. + - To hide a stage, select **Hide** (**{eye-slash}**). +1. To add a custom stage, select **Add another stage**. + - Enter a name for the stage. + - Select a **Start event** and a **Stop event**. +1. Select **Create value stream**. NOTE: If you have recently upgraded to GitLab Premium, it can take up to 30 minutes for data to collect and display. -### Create a value stream with stages +## Create a value stream with custom stages > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50229) in GitLab 13.7. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55572) in GitLab 13.10. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/294190) in GitLab 13.11. -You can create value streams with stages, starting with a default or a blank template. You can -add stages as desired. - -To create a value stream with stages: +When you create a value stream, you can create and add custom stages that align with your own development workflows. 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. -1. If this is the first time you are creating a value stream, select **Create custom value stream**. Otherwise, in the top right, select the dropdown list and then **Create new Value Stream**. -1. Select either **Create from default template** or **Create from no template**. - - You can hide or re-order default stages in the value stream. - -  - - - You can add new stages by selecting **Add another stage**. - - You can select the name and start and end events for the stage. - -  -1. Select **Create Value Stream**. - -### Label-based stages - -The pre-defined start and end events can cover many use cases involving both issues and merge requests. +1. Select **Create value stream**. +1. For each stage: + - Enter a name for the stage. + - Select a **Start event** and a **Stop event**. +1. To add another stage, select **Add another stage**. +1. To re-order the stages, select the up or down arrows. +1. Select **Create value stream**. -In more complex workflows, use stages based on group labels. These events are based on -added or removed labels. In particular, [scoped labels](../../project/labels.md#scoped-labels) -are useful for complex workflows. +### Label-based stages for custom value streams -In this example, we'd like to measure times for deployment from a staging environment to production. The workflow is the following: +To measure complex workflows, you can use [scoped labels](../../project/labels.md#scoped-labels). For example, to measure deployment +time from a staging environment to production, you could use the following labels: - When the code is deployed to staging, the `workflow::staging` label is added to the merge request. - When the code is deployed to production, the `workflow::production` label is added to the merge request.  -### Edit a value stream +## Edit a value stream > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267537) in GitLab 13.10. @@ -342,19 +319,18 @@ After you create a value stream, you can customize it to suit your purposes. To 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. -1. In the top right, select the dropdown list and then select the relevant value stream. +1. In the top right, select the dropdown list, and select a value stream. 1. Next to the value stream dropdown list, select **Edit**. - The edit form is populated with the value stream details. 1. Optional: - Rename the value stream. - Hide or re-order default stages. - Remove existing custom stages. - - Add new stages by selecting the 'Add another stage' button + - To add new stages, select **Add another stage**. - Select the start and end events for the stage. 1. Optional. To undo any modifications, select **Restore value stream defaults**. 1. Select **Save Value Stream**. -### Delete a value stream +## Delete a value stream > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221205) in GitLab 13.4. diff --git a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md index ab04187284d..be58b4565a1 100644 --- a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md @@ -97,6 +97,8 @@ Use CI/CD environment variables to configure your project. 1. Expand **Variables**. 1. Set the variable `BASE64_GOOGLE_CREDENTIALS` to the `base64` encoded JSON file you just created. 1. Set the variable `TF_VAR_gcp_project` to your GCP's `project` name. +1. Set the variable `TF_VAR_agent_token` to the agent token displayed in the previous task. +1. Set the variable `TF_VAR_kas_address` to the agent server address displayed in the previous task. **Optional configuration:** diff --git a/doc/user/infrastructure/clusters/index.md b/doc/user/infrastructure/clusters/index.md index cd7c5feb284..933b310ff3f 100644 --- a/doc/user/infrastructure/clusters/index.md +++ b/doc/user/infrastructure/clusters/index.md @@ -18,7 +18,7 @@ as well as its related [features](#deprecated-features). The certificate-based Kubernetes integration with GitLab is deprecated. It had the following issues: -- There were security issues as it required direct access to the Kube API by GitLab. +- There were security issues as it required direct access to the Kubernetes API by GitLab. - The configuration options weren't flexible. - The integration was flaky. - Users were constantly reporting issues with features based on this model. @@ -42,22 +42,18 @@ the GitLab agent model on the [agent's blueprint documentation](../../../archite ## Deprecated features -- [Create a new cluster through cluster certificates](../../project/clusters/add_remove_clusters.md) - [Connect an existing cluster through cluster certificates](../../project/clusters/add_existing_cluster.md) - [Access controls](../../project/clusters/cluster_access.md) - [GitLab-managed clusters](../../project/clusters/gitlab_managed_clusters.md) -- [GitLab Managed Apps](../../clusters/applications.md) - [Deploy applications through certificate-based connection](../../project/clusters/deploy_to_cluster.md) - [Cluster Management Project](../../clusters/management_project.md) - [Cluster integrations](../../clusters/integrations.md) - [Cluster cost management](../../clusters/cost_management.md) - [Cluster environments](../../clusters/environments.md) -- [Advanced traffic control with Canary Ingress](../../project/canary_deployments.md#advanced-traffic-control-with-canary-ingress-deprecated) -- [Serverless](../../project/clusters/serverless/index.md) +- [Show Canary Ingress deployments on deploy boards](../../project/canary_deployments.md#show-canary-ingress-deployments-on-deploy-boards-deprecated) - [Deploy Boards](../../project/deploy_boards.md) - [Pod logs](../../project/clusters/kubernetes_pod_logs.md) - [Clusters health](manage/clusters_health.md) -- [Crossplane integration](../../clusters/crossplane.md) - [Web terminals](../../../administration/integration/terminal.md) ### Cluster levels @@ -67,5 +63,5 @@ The concept of [project-level](../../project/clusters/index.md), [instance-level](../../instance/clusters/index.md) clusters becomes extinct in the new model, although the functionality remains to some extent. -The agent is always configured in a single GitLab project and you can expose the cluster connection to other projects and groups to [access it from GitLab CI/CD](../../clusters/agent/ci_cd_tunnel.md). +The agent is always configured in a single GitLab project and you can expose the cluster connection to other projects and groups to [access it from GitLab CI/CD](../../clusters/agent/ci_cd_workflow.md). By doing so, you are granting these projects and groups access to the same cluster, which is similar to group-level clusters' use case. diff --git a/doc/user/infrastructure/clusters/manage/clusters_health.md b/doc/user/infrastructure/clusters/manage/clusters_health.md index eeb931f392f..0eefae3f550 100644 --- a/doc/user/infrastructure/clusters/manage/clusters_health.md +++ b/doc/user/infrastructure/clusters/manage/clusters_health.md @@ -4,10 +4,15 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Clusters health **(FREE)** +# Clusters health (DEPRECATED) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4701) in GitLab 10.6. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) from GitLab Ultimate to GitLab Free in 13.2. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. + +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. However, you can **still use** Prometheus +for Kubernetes clusters connected to GitLab by [enabling Prometheus manually](../../../project/integrations/prometheus.md#manual-configuration-of-prometheus). When [the Prometheus cluster integration is enabled](../../../clusters/integrations.md#prometheus-cluster-integration), GitLab monitors the cluster's health. At the top of the cluster settings page, CPU and Memory utilization is displayed, along with the total amount available. Keeping an eye on cluster resources can be important, if the cluster runs out of memory pods may be shutdown or fail to start. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/apparmor.md b/doc/user/infrastructure/clusters/manage/management_project_applications/apparmor.md deleted file mode 100644 index ae335a180e8..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/apparmor.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Install AppArmor with a cluster management project **(FREE)** - -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install AppArmor you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/apparmor/helmfile.yaml -``` - -You can define one or more AppArmor profiles by adding them into -`applications/apparmor/values.yaml` as the following: - -```yaml -profiles: - profile-one: |- - profile profile-one { - file, - } -``` - -Refer to the [AppArmor chart](https://gitlab.com/gitlab-org/charts/apparmor) for more information on this chart. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md index 58de5f5e368..5ad1fb81a39 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - Support for cert-manager v1.4 was [introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/69405) in GitLab 14.3. > - [Upgraded](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/23) to cert-manager 1.7 in GitLab 14.8. -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install cert-manager you should uncomment this line from your `helmfile.yaml`: diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md b/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md deleted file mode 100644 index 5d704a2c6df..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/cilium.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Install Cilium with a cluster management project **(FREE)** - -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -[Cilium](https://cilium.io/) is a networking plugin for Kubernetes that you can use to implement -support for [NetworkPolicy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -resources. For more information, see [Network Policies](../../../../../topics/autodevops/stages.md#network-policy). - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview, see the -[Container Network Security Demo for GitLab 12.8](https://www.youtube.com/watch?v=pgUEdhdhoUI). - -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install cilium you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/cilium/helmfile.yaml -``` - -and update the `applications/cilium/values.yaml` to set the `clusterType`: - -```yaml -# possible values are gke or eks -clusterType: gke -``` - -The `clusterType` variable enables the recommended Helm variables for a corresponding cluster type. -You can check the recommended variables for each cluster type in the official documentation: - -- [Google GKE](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-gke/#deploy-cilium) -- [AWS EKS](https://docs.cilium.io/en/v1.8/gettingstarted/k8s-install-eks/#deploy-cilium) - -Do not use `clusterType` for sandbox environments like [minikube](https://minikube.sigs.k8s.io/docs/). - -You can customize Cilium's Helm variables by defining the -`applications/cilium/values.yaml` file in your cluster -management project. Refer to the -[Cilium chart](https://github.com/cilium/cilium/tree/master/install/kubernetes/cilium) -for the available configuration options. - -You can check Cilium's installation status on the cluster management page: - -- [Project-level cluster](../../../../project/clusters/index.md): Navigate to your project's - **Infrastructure > Kubernetes clusters** page. -- [Group-level cluster](../../../../group/clusters/index.md): Navigate to your group's - **Kubernetes** page. - -WARNING: -Installation and removal of the Cilium requires a **manual** -[restart](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-helm/#restart-unmanaged-pods) -of all affected pods in all namespaces to ensure that they are -[managed](https://docs.cilium.io/en/v1.8/operations/troubleshooting/#ensure-managed-pod) -by the correct networking plugin. Whenever Hubble is enabled, its related pod might require a -restart depending on whether it started prior to Cilium. For more information, see -[Failed Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#failed-deployment) -in the Kubernetes docs. - -NOTE: -Major upgrades might require additional setup steps. For more information, see -the official [upgrade guide](https://docs.cilium.io/en/v1.8/operations/upgrade/). - -By default, Cilium's -[audit mode](https://docs.cilium.io/en/v1.8/gettingstarted/policy-creation/#enable-policy-audit-mode) -is enabled. In audit mode, Cilium doesn't drop disallowed packets. You -can use `policy-verdict` log to observe policy-related decisions. You -can disable audit mode by adding the following to -`applications/cilium/values.yaml`: - -```yaml -config: - policyAuditMode: false - -agent: - monitor: - eventTypes: ["drop"] -``` - -The Cilium monitor log for traffic is logged out by the -`cilium-monitor` sidecar container. You can check these logs with the following command: - -```shell -kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor -``` - -You can disable the monitor log in `.gitlab/managed-apps/cilium/values.yaml`: - -```yaml -agent: - monitor: - enabled: false -``` - -The [Hubble](https://github.com/cilium/hubble) monitoring daemon is enabled by default -and it's set to collect per namespace flow metrics. This metrics are accessible on the -[Threat Monitoring](../../../../application_security/threat_monitoring/index.md) -dashboard. You can disable Hubble by adding the following to -`applications/cilium/values.yaml`: - -```yaml -global: - hubble: - enabled: false -``` - -You can also adjust Helm values for Hubble by using -`applications/cilium/values.yaml`: - -```yaml -global: - hubble: - enabled: true - metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' -``` diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/elasticstack.md b/doc/user/infrastructure/clusters/manage/management_project_applications/elasticstack.md index f9d0948a2bb..7ab99ab3875 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/elasticstack.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/elasticstack.md @@ -2,28 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: '../../index.md' --- -# Install Elastic Stack with a cluster management project **(FREE)** +# Install Elastic Stack with a cluster management project (removed) **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install Elastic Stack you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/elastic-stack/helmfile.yaml -``` - -Elastic Stack is installed by default into the `gitlab-managed-apps` namespace of your cluster. - -You can check the default -[`values.yaml`](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/blob/master/applications/elastic-stack/values.yaml) -we set for this chart. - -You can customize the installation of Elastic Stack by updating the -`applications/elastic-stack/values.yaml` file in your cluster -management project. Refer to the -[chart](https://gitlab.com/gitlab-org/charts/elastic-stack) for all -available configuration options. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 14.8 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/360182) in 15.0. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/falco.md b/doc/user/infrastructure/clusters/manage/management_project_applications/falco.md deleted file mode 100644 index 50401e9a391..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/falco.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Install Falco with a cluster management project **(FREE)** - -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -GitLab Container Host Security Monitoring uses [Falco](https://falco.org/) -as a runtime security tool that listens to the Linux kernel using eBPF. Falco parses system calls -and asserts the stream against a configurable rules engine in real-time. For more information, see -[Falco's Documentation](https://falco.org/docs/). - -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install Falco you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/falco/helmfile.yaml -``` - -You can customize Falco's Helm variables by defining the -`applications/falco/values.yaml` file in your cluster -management project. Refer to the -[Falco chart](https://github.com/falcosecurity/charts/tree/master/falco) -for the available configuration options. - -WARNING: -By default eBPF support is enabled and Falco uses an -[eBPF probe](https://falco.org/docs/event-sources/drivers/#using-the-ebpf-probe) -to pass system calls to user space. If your cluster doesn't support this, you can -configure it to use Falco kernel module instead by adding the following to -`applications/falco/values.yaml`: - -```yaml -ebpf: - enabled: false -``` - -In rare cases where probe installation on your cluster isn't possible and the kernel/probe -isn't pre-compiled, you may need to manually prepare the kernel module or eBPF probe with -[`driverkit`](https://github.com/falcosecurity/driverkit#against-a-kubernetes-cluster) -and install it on each cluster node. - -By default, Falco is deployed with a limited set of rules. To add more rules, add -the following to `applications/falco/values.yaml` (you can get examples from -[Cloud Native Security Hub](https://securityhub.dev/)): - -```yaml -customRules: - file-integrity.yaml: |- - - rule: Detect New File - desc: detect new file created - condition: > - evt.type = chmod or evt.type = fchmod - output: > - File below a known directory opened for writing (user=%user.name - command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) - priority: ERROR - tags: [filesystem] - - rule: Detect New Directory - desc: detect new directory created - condition: > - mkdir - output: > - File below a known directory opened for writing (user=%user.name - command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) - priority: ERROR - tags: [filesystem] -``` - -By default, Falco only outputs security events to logs as JSON objects. To set it to output to an -[external API](https://falco.org/docs/alerts/#https-output-send-alerts-to-an-https-end-point) -or [application](https://falco.org/docs/alerts/#program-output), -add the following to `applications/falco/values.yaml`: - -```yaml -falco: - programOutput: - enabled: true - keepAlive: false - program: mail -s "Falco Notification" someone@example.com - - httpOutput: - enabled: true - url: http://some.url -``` - -You can check these logs with the following command: - -```shell -kubectl -n gitlab-managed-apps logs -l app=falco -``` diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/fluentd.md b/doc/user/infrastructure/clusters/manage/management_project_applications/fluentd.md deleted file mode 100644 index ea3a3503f9b..00000000000 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/fluentd.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Install Fluentd with a cluster management project **(FREE)** - -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install Fluentd you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/fluentd/helmfile.yaml -``` - -You can also review the default values set for this chart in the -[`values.yaml`](https://github.com/helm/charts/blob/master/stable/fluentd/values.yaml) file. - -You can customize the installation of Fluentd by defining -`applications/fluentd/values.yaml` file in your cluster management -project. Refer to the -[configuration chart](https://github.com/helm/charts/tree/master/stable/fluentd#configuration) -for the current development release of Fluentd for all available configuration options. - -The configuration chart link points to the current development release, which -may differ from the version you have installed. To ensure compatibility, switch -to the specific branch or tag you are using. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md b/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md index 503f077df14..7983a640577 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install Ingress you should uncomment this line from your `helmfile.yaml`: diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md b/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md index f76c7363a83..383e857bb20 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md @@ -12,7 +12,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w open-source monitoring and alerting system for supervising your deployed applications. -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install Prometheus you should uncomment this line from your `helmfile.yaml`: diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/runner.md b/doc/user/infrastructure/clusters/manage/management_project_applications/runner.md index 4faf5f46418..ef7c4637607 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/runner.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/runner.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install GitLab Runner you should uncomment this line from your `helmfile.yaml`: @@ -35,7 +35,7 @@ These values can be specified using [CI/CD variables](../../../../../ci/variable The methods of specifying these values are mutually exclusive. Either specify variables `GITLAB_RUNNER_REGISTRATION_TOKEN` and `CI_SERVER_URL` as CI variables (recommended) or provide values for `runnerRegistrationToken:` and `gitlabUrl:` in `applications/gitlab-runner/values.yaml.gotmpl`. -The runner registration token allows connection to a project by a runner and therefore should be treated as a secret to prevent malicious use and code exfiltration through a runner. For this reason, we recommend that you specify the runner registration token as a [protected variable](../../../../../ci/variables/index.md#protect-a-cicd-variable) and [masked variable](../../../../../ci/variables/index.md#mask-a-cicd-variable) and do not commit them to the Git repository in the `values.yaml.gotmpl` file. +The runner registration token allows connection to a project by a runner and therefore should be treated as a secret to prevent malicious use and code exfiltration through a runner. For this reason, we recommend that you specify the runner registration token as a [protected variable](../../../../../ci/variables/index.md#protected-cicd-variables) and [masked variable](../../../../../ci/variables/index.md#mask-a-cicd-variable) and do not commit them to the Git repository in the `values.yaml.gotmpl` file. You can customize the installation of GitLab Runner by defining `applications/gitlab-runner/values.yaml.gotmpl` file in your cluster diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md b/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md index b968e63d632..d2d314b649e 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w The Sentry Helm chart [recommends](https://github.com/helm/charts/blob/f6e5784f265dd459c5a77430185d0302ed372665/stable/sentry/values.yaml#L284-L285) at least 3 GB of available RAM for database migrations. -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install Sentry you should uncomment this line from your `helmfile.yaml`: diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md index 4618a95f986..06e67b78c91 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md @@ -20,7 +20,7 @@ control. Therefore, if GitLab is compromised, the security of this Vault instanc avoid this security risk, GitLab recommends using your own HashiCorp Vault to leverage [external secrets with CI](../../../../../ci/secrets/index.md). -Assuming you already have a [Cluster management project](../../../../../user/clusters/management_project.md) created from a +Assuming you already have a project created from a [management project template](../../../../../user/clusters/management_project_template.md), to install Vault you should uncomment this line from your `helmfile.yaml`: diff --git a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md index 61ec0a559f0..7b2b5b4afd4 100644 --- a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md +++ b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md @@ -9,19 +9,20 @@ info: To determine the technical writer assigned to the Stage/Group associated w To connect your Kubernetes cluster with GitLab, you can use: - [A GitOps workflow](../../clusters/agent/gitops.md). -- [A GitLab CI/CD workflow](../../clusters/agent/ci_cd_tunnel.md). +- [A GitLab CI/CD workflow](../../clusters/agent/ci_cd_workflow.md). - [A certificate-based integration](index.md). The certificate-based integration is [**deprecated**](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/) -in GitLab 14.5. It is expected to be -[turned off by default in 15.0](../../../update/deprecations.md#certificate-based-integration-with-kubernetes) -and removed in GitLab 15.6. +in GitLab 14.5. The sunsetting plans are described: + +- for [GitLab.com customers](../../../update/deprecations.md#saas-certificate-based-integration-with-kubernetes). +- for [Self-managed customers](../../../update/deprecations.md#self-managed-certificate-based-integration-with-kubernetes). If you are using the certificate-based integration, you should move to another workflow as soon as possible. As a general rule, to migrate clusters that rely on GitLab CI/CD, -you can use the [CI/CD workflow](../../clusters/agent/ci_cd_tunnel.md). +you can use the [CI/CD workflow](../../clusters/agent/ci_cd_workflow.md). This workflow uses an agent to connect to your cluster. The agent: - Is not exposed to the internet. @@ -39,7 +40,7 @@ Some features are currently available only when using certificate-based integrat With GitLab-managed clusters, GitLab creates separate service accounts and namespaces for every branch and deploys by using these resources. -The GitLab agent uses [impersonation](../../clusters/agent/ci_cd_tunnel.md#use-impersonation-to-restrict-project-and-group-access) +The GitLab agent uses [impersonation](../../clusters/agent/ci_cd_workflow.md#use-impersonation-to-restrict-project-and-group-access) strategies to deploy to your cluster with restricted account access. To do so: 1. Choose the impersonation strategy that suits your needs. @@ -48,30 +49,55 @@ strategies to deploy to your cluster with restricted account access. To do so: ### Migrate from Auto DevOps -To configure your Auto DevOps project to use the GitLab agent: - -1. Follow the steps to [install an agent](../../clusters/agent/install/index.md) in your cluster. -1. Go to the project where you use Auto DevOps. -1. On the left sidebar, select **Settings > CI/CD** and expand **Variables**. -1. Select **Add new variable**. -1. Add `KUBE_CONTEXT` as the key, `path/to/agent/project:agent-name` as the value, and select the environment scope of your choice. +In your Auto DevOps project, you can use the GitLab agent to connect with your Kubernetes cluster. + +1. [Install an agent](../../clusters/agent/install/index.md) in your cluster. +1. In GitLab, go to the project where you use Auto DevOps. +1. Add three variables. On the left sidebar, select **Settings > CI/CD** and expand **Variables**. + - Add a key called `KUBE_INGRESS_BASE_DOMAIN` with the application deployment domain as the value. + - Add a key called `KUBE_CONTEXT` with a value like `path/to/agent/project:agent-name`. + Select the environment scope of your choice. + If you are not sure what your agent’s context is, edit your `.gitlab-ci.yml` file and add a job to see the available contexts: + + ```yaml + deploy: + image: + name: bitnami/kubectl:latest + entrypoint: [""] + script: + - kubectl config get-contexts + ``` + + - Add a key called `KUBE_NAMESPACE` with a value of the Kubernetes namespace for your deployments to target. Set the same environment scope. 1. Select **Add variable**. -1. Repeat the process to add another variable, `KUBE_NAMESPACE`, setting the value for the Kubernetes namespace you want your deployments to target, and set the same environment scope from the previous step. 1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. 1. From the certificate-based clusters section, open the cluster that serves the same environment scope. 1. Select the **Details** tab and disable the cluster. -1. To activate the changes, on the left sidebar, select **CI/CD > Pipelines** and then **Run pipeline**. +1. Edit your `.gitlab-ci.yml` file and ensure it's using the Auto DevOps template. For example: + + ```yaml + include: + template: Auto-DevOps.gitlab-ci.yml + + variables: + KUBE_INGRESS_BASE_DOMAIN: 74.220.23.215.nip.io + KUBE_CONTEXT: "gitlab-examples/ops/gitops-demo/k8s-agents:demo-agent" + KUBE_NAMESPACE: "demo-agent" + ``` + +1. To test your pipeline, on the left sidebar, select **CI/CD > Pipelines** and then **Run pipeline**. For an example, [view this project](https://gitlab.com/gitlab-examples/ops/gitops-demo/hello-world-service). ### Migrate generic deployments -Follow the process for the [CI/CD workflow](../../clusters/agent/ci_cd_tunnel.md). +Follow the process for the [CI/CD workflow](../../clusters/agent/ci_cd_workflow.md). ## Migrate from GitLab Managed applications -[GitLab Managed Apps (GMA)](../../clusters/applications.md#gitlab-managed-apps-deprecated) were deprecated in GitLab 14.0, and -the agent for Kubernetes does not support them. To migrate from GMA to the agent, go through the following steps: +GitLab Managed Apps (GMA) were deprecated in GitLab 14.0, and removed in GitLab 15.0. +The agent for Kubernetes does not support them. To migrate from GMA to the +agent, go through the following steps: 1. [Migrate from GitLab Managed Apps to a cluster management project](../../clusters/migrating_from_gma_to_project_template.md). 1. [Migrate the cluster management project to use the agent](../../clusters/management_project_template.md). diff --git a/doc/user/infrastructure/iac/img/terraform_list_view_v13_8.png b/doc/user/infrastructure/iac/img/terraform_list_view_v13_8.png Binary files differdeleted file mode 100644 index 6eb85285e81..00000000000 --- a/doc/user/infrastructure/iac/img/terraform_list_view_v13_8.png +++ /dev/null diff --git a/doc/user/infrastructure/iac/index.md b/doc/user/infrastructure/iac/index.md index bc7a3c0d069..0f7965a3527 100644 --- a/doc/user/infrastructure/iac/index.md +++ b/doc/user/infrastructure/iac/index.md @@ -86,7 +86,7 @@ To use a Terraform template: # TF_ROOT: terraform/production ``` -1. (Optional) Override in your `.gitlab-ci.yaml` file the attributes present +1. (Optional) Override in your `.gitlab-ci.yml` file the attributes present in the template you fetched to customize your configuration. ## GitLab-managed Terraform state @@ -107,13 +107,9 @@ workflows. ## The GitLab Terraform provider -NOTE: -The GitLab Terraform provider is released separately from GitLab. -We are working on migrating the GitLab Terraform provider to GitLab.com. - -The [GitLab Terraform provider](https://github.com/gitlabhq/terraform-provider-gitlab) is a plugin for Terraform to facilitate -managing of GitLab resources such as users, groups, and projects. -Its documentation is available on [Terraform](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs). +The [GitLab Terraform provider](https://github.com/gitlabhq/terraform-provider-gitlab) is a Terraform plugin to facilitate +managing of GitLab resources such as users, groups, and projects. It is released separately from GitLab +and its documentation is available on [Terraform](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs). ## Create a new cluster through IaC diff --git a/doc/user/infrastructure/iac/terraform_state.md b/doc/user/infrastructure/iac/terraform_state.md index 60f97f522cf..f56fe92ec01 100644 --- a/doc/user/infrastructure/iac/terraform_state.md +++ b/doc/user/infrastructure/iac/terraform_state.md @@ -8,58 +8,43 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. -[Terraform remote backends](https://www.terraform.io/language/settings/backends) -enable you to store the state file in a remote, shared store. GitLab uses the -[Terraform HTTP backend](https://www.terraform.io/language/settings/backends/http) -to securely store the state files in local storage (the default) or -[the remote store of your choice](../../../administration/terraform_state.md). +Terraform uses state files to store details about your infrastructure configuration. +With Terraform remote [backends](https://www.terraform.io/language/settings/backends), +you can store the state file in a remote and shared store. -WARNING: -Using local storage (the default) on clustered deployments of GitLab will result in -a split state across nodes, making subsequent executions of Terraform inconsistent. -You are highly advised to use a remote storage resource in that case. - -The GitLab-managed Terraform state backend can store your Terraform state easily and -securely, and spares you from setting up additional remote resources like -Amazon S3 or Google Cloud Storage. Its features include: - -- Versioning of Terraform state files. -- Supporting encryption of the state file both in transit and at rest. -- Locking and unlocking state. -- Remote Terraform plan and apply execution. - -A GitLab **administrator** must [set up the Terraform state storage configuration](../../../administration/terraform_state.md) -before using this feature. +GitLab provides a [Terraform HTTP backend](https://www.terraform.io/language/settings/backends/http) +to securely store your state files with minimal configuration. -## Permissions for using Terraform +In GitLab, you can: -In GitLab version 13.1, at least the Maintainer role was required to use a -GitLab managed Terraform state backend. +- Version your Terraform state files. +- Encrypt the state file both in transit and at rest. +- Lock and unlock states. +- Remotely execute `terraform plan` and `terraform apply` commands. -In GitLab versions 13.2 and later, at least: +For self-managed instances, before you can use GitLab for your Terraform state files, +an administrator must [set up Terraform state storage](../../../administration/terraform_state.md). -- The Maintainer role is required to lock, unlock, and write to the state (using `terraform apply`). -- The Developer role is required to read the state (using `terraform plan -lock=false`). +## Initialize a Terraform state as a backend by using GitLab CI/CD -## Set up GitLab-managed Terraform state +After you execute the `terraform init` command, you can use GitLab CI/CD +to run `terraform` commands. -To get started with a GitLab-managed Terraform state, there are two different options: +Prerequisites: -- [Use a local machine](#get-started-using-local-development). -- [Use GitLab CI](#get-started-using-gitlab-ci). +- To lock, unlock, and write to the state by using `terraform apply`, you must have at least the Maintainer role. +- To read the state by using `terraform plan -lock=false`, you must have at least the Developer role. -Terraform States can be found by navigating to a Project's -**{cloud-gear}** **Infrastructure > Terraform** page. - -### Get started using local development +WARNING: +Like any other job artifact, Terraform plan data is viewable by anyone with the Guest role on the repository. +Neither Terraform nor GitLab encrypts the plan file by default. If your Terraform plan +includes sensitive data, like passwords, access tokens, or certificates, you should +encrypt plan output or modify the project visibility settings. -If you plan to only run `terraform plan` and `terraform apply` commands from your -local machine, this is a simple way to get started: +To configure GitLab CI/CD as a backend: -1. Create your project on your GitLab instance. -1. Navigate to **Settings > General** and note your **Project name** - and **Project ID**. -1. Define the Terraform backend in your Terraform project to be: +1. In your Terraform project, in a `.tf` file like `backend.tf`, + define the [HTTP backend](https://www.terraform.io/docs/language/settings/backends/http.html): ```hcl terraform { @@ -68,172 +53,51 @@ local machine, this is a simple way to get started: } ``` -1. Create a [Personal Access Token](../../profile/personal_access_tokens.md) with - the `api` scope. - -1. On your local machine, run `terraform init`, passing in the following options, - replacing `<YOUR-STATE-NAME>`, `<YOUR-PROJECT-ID>`, `<YOUR-USERNAME>` and - `<YOUR-ACCESS-TOKEN>` with the relevant values. This command initializes your - Terraform state, and stores that state in your GitLab project. This example - uses `gitlab.com`: - - ```shell - terraform init \ - -backend-config="address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>" \ - -backend-config="lock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>/lock" \ - -backend-config="unlock_address=https://gitlab.com/api/v4/projects/<YOUR-PROJECT-ID>/terraform/state/<YOUR-STATE-NAME>/lock" \ - -backend-config="username=<YOUR-USERNAME>" \ - -backend-config="password=<YOUR-ACCESS-TOKEN>" \ - -backend-config="lock_method=POST" \ - -backend-config="unlock_method=DELETE" \ - -backend-config="retry_wait_min=5" - ``` - - WARNING: - The name of your state can contain only uppercase and lowercase letters, decimal digits, - hyphens, and underscores. - -If you already have a GitLab-managed Terraform state, you can use the `terraform init` command -with the pre-populated parameters values: - -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Infrastructure > Terraform**. -1. Next to the environment you want to use, select the [Actions menu](#managing-state-files) - **{ellipsis_v}** and select **Copy Terraform init command**. - -You can now run `terraform plan` and `terraform apply` as you normally would. - -### Get started using GitLab CI - -If you don't want to start with local development, you can also use GitLab CI to -run your `terraform plan` and `terraform apply` commands. - -Next, [configure the backend](#configure-the-backend). - -#### Configure the backend +1. In the root directory of your project repository, create a `.gitlab-ci.yml` file. Use + [this file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) + to populate it. + +1. Push your project to GitLab. This action triggers a pipeline, which + runs the `gitlab-terraform init`, `gitlab-terraform validate`, and + `gitlab-terraform plan` commands. +1. Trigger the manual `terraform apply` job from the previous pipeline to provision the defined infrastructure. -After executing the `terraform init` command, you must configure the Terraform backend -and the CI YAML file: +The output from the above `terraform` commands should be viewable in the job logs. -1. In your Terraform project, define the [HTTP backend](https://www.terraform.io/docs/language/settings/backends/http.html) - by adding the following code block in a `.tf` file (such as `backend.tf`) to - define the remote backend: +## Access the state from your local machine - ```hcl - terraform { - backend "http" { - } - } - ``` +You can access the GitLab-managed Terraform state from your local machine. -1. In the root directory of your project repository, configure a - `.gitlab-ci.yml` file. This example uses a pre-built image which includes a - `gitlab-terraform` helper. For supported Terraform versions, see the [GitLab - Terraform Images project](https://gitlab.com/gitlab-org/terraform-images). +WARNING: +On clustered deployments of GitLab, you should not use local storage. +A split state can occur across nodes, making subsequent Terraform executions +inconsistent. Instead, use a remote storage resource. - ```yaml - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - ``` +1. Ensure the Terraform state has been + [initialized for CI/CD](#initialize-a-terraform-state-as-a-backend-by-using-gitlab-cicd). +1. Copy a pre-populated Terraform `init` command: -1. In the `.gitlab-ci.yml` file, define some CI/CD variables to ease - development. In this example, `TF_ROOT` is the directory where the Terraform - commands must be executed, `TF_ADDRESS` is the URL to the state on the GitLab - instance where this pipeline runs, and the final path segment in `TF_ADDRESS` - is the name of the Terraform state. Projects may have multiple states, and - this name is arbitrary, so in this example we set it to `example-production` - which corresponds with the directory we're using as our `TF_ROOT`, and we - ensure that the `.terraform` directory is cached between jobs in the pipeline - using a cache key based on the state name (`example-production`): - - ```yaml - variables: - TF_ROOT: ${CI_PROJECT_DIR}/environments/example/production - TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/example-production - - cache: - key: example-production - paths: - - ${TF_ROOT}/.terraform - ``` + 1. On the top bar, select **Menu > Projects** and find your project. + 1. On the left sidebar, select **Infrastructure > Terraform**. + 1. Next to the environment you want to use, select **Actions** + (**{ellipsis_v}**) and select **Copy Terraform init command**. -1. In a `before_script`, change to your `TF_ROOT`: - - ```yaml - before_script: - - cd ${TF_ROOT} - - stages: - - prepare - - validate - - build - - deploy - - init: - stage: prepare - script: - - gitlab-terraform init - - validate: - stage: validate - script: - - gitlab-terraform validate - - plan: - stage: build - script: - - gitlab-terraform plan - - gitlab-terraform plan-json - artifacts: - name: plan - paths: - - ${TF_ROOT}/plan.cache - reports: - terraform: ${TF_ROOT}/plan.json - - apply: - stage: deploy - environment: - name: production - script: - - gitlab-terraform apply - dependencies: - - plan - when: manual - only: - - master - ``` - -1. Push your project to GitLab, which triggers a CI job pipeline. This pipeline - runs the `gitlab-terraform init`, `gitlab-terraform validate`, and - `gitlab-terraform plan` commands. +1. Open a terminal and run this command on your local machine. -The output from the above `terraform` commands should be viewable in the job logs. +## Migrate to a GitLab-managed Terraform state -WARNING: -Like any other job artifact, Terraform plan data is viewable by anyone with the Guest role on the repository. -Neither Terraform nor GitLab encrypts the plan file by default. If your Terraform plan -includes sensitive data such as passwords, access tokens, or certificates, GitLab strongly -recommends encrypting plan output or modifying the project visibility settings. +Terraform supports copying the state when the backend changes or is +reconfigured. Use these actions to migrate from another backend to +GitLab-managed Terraform state. -### Example project +You should use a local terminal to run the commands needed for migrating to GitLab-managed Terraform state. -See [this reference project](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-aws) using GitLab and Terraform to deploy a basic AWS EC2 in a custom VPC. +The following example demonstrates how to change the state name. The same workflow is needed to migrate to GitLab-managed Terraform state from a different state storage backend. -## Using a GitLab-managed Terraform state backend as a remote data source +## Use your GitLab backend as a remote data source -You can use a GitLab-managed Terraform state as a +You can use a GitLab-managed Terraform state backend as a [Terraform data source](https://www.terraform.io/language/state/remote-state-data). -To use your existing Terraform state backend as a data source, provide the following details -as [Terraform input variables](https://www.terraform.io/language/values/variables): - -- **address**: The URL of the remote state backend you want to use as a data source. - For example, `https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>`. -- **username**: The username to authenticate with the data source. If you are using a [Personal Access Token](../../profile/personal_access_tokens.md) for - authentication, this is your GitLab username. If you are using GitLab CI, this is `'gitlab-ci-token'`. -- **password**: The password to authenticate with the data source. If you are using a Personal Access Token for - authentication, this is the token value. If you are using GitLab CI, it is the contents of the `${CI_JOB_TOKEN}` CI/CD variable. - -An example setup is shown below: 1. Create a file named `example.auto.tfvars` with the following contents: @@ -243,7 +107,7 @@ An example setup is shown below: example_access_token=<GitLab Personal Access Token> ``` -1. Define the data source by adding the following code block in a `.tf` file (such as `data.tf`): +1. In a `.tf` file, define the data source by using [Terraform input variables](https://www.terraform.io/language/values/variables): ```hcl data "terraform_remote_state" "example" { @@ -257,21 +121,20 @@ An example setup is shown below: } ``` + - **address**: The URL of the remote state backend you want to use as a data source. + For example, `https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>`. + - **username**: The username to authenticate with the data source. If you are using + a [Personal Access Token](../../profile/personal_access_tokens.md) for + authentication, this value is your GitLab username. If you are using GitLab CI/CD, this value is `'gitlab-ci-token'`. + - **password**: The password to authenticate with the data source. If you are using a Personal Access Token for + authentication, this value is the token value. If you are using GitLab CI/CD, this value is the contents of the `${CI_JOB_TOKEN}` CI/CD variable. + Outputs from the data source can now be referenced in your Terraform resources using `data.terraform_remote_state.example.outputs.<OUTPUT-NAME>`. -You need at least the Developer role in the target project -to read the Terraform state. +To read the Terraform state in the target project, you need at least the Developer role. -## Migrating to GitLab-managed Terraform state - -Terraform supports copying the state when the backend is changed or -reconfigured. This can be useful if you need to migrate from another backend to -GitLab-managed Terraform state. Using a local terminal is recommended to run the commands needed for migrating to GitLab-managed Terraform state. - -The following example demonstrates how to change the state name, the same workflow is needed to migrate to GitLab-managed Terraform state from a different state storage backend. - -### Setting up the initial backend +### Set up the initial backend ```shell PROJECT_ID="<gitlab-project-id>" @@ -309,7 +172,7 @@ re-run this command to reinitialize your working directory. If you forget, other commands will detect it and remind you to do so if necessary. ``` -### Changing the backend +### Change the backend Now that `terraform init` has created a `.terraform/` directory that knows where the old state is, you can tell it about the new location: @@ -366,94 +229,54 @@ commands will detect it and remind you to do so if necessary. If you type `yes`, it copies your state from the old location to the new location. You can then go back to running it in GitLab CI/CD. -## Managing state files +## Manage Terraform state files > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/273592) in GitLab 13.8. -Users with at least the Developer role can view the -state files attached to a project at **Infrastructure > Terraform**. Users with the -Maintainer role can perform commands on the state files. The user interface -contains these fields: - - +To view Terraform state files: -- **Name**: The name of the environment, with a locked (**{lock}**) icon if the - state file is locked. -- **Pipeline**: A link to the most recent pipeline and its status. -- **Details**: Information about when the state file was created or changed. -- **Actions**: Actions you can take on the state file, including copying the `terraform init` command, - downloading, locking, unlocking, or [removing](#remove-a-state-file) the state file and versions. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Infrastructure > Terraform**. -NOTE: -Additional improvements to the -[graphical interface for managing state files](https://gitlab.com/groups/gitlab-org/-/epics/4563) -are planned. +[An epic exists](https://gitlab.com/groups/gitlab-org/-/epics/4563) to track improvements to this UI. -## Manage individual Terraform state versions +### Manage individual Terraform state versions > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207347) in GitLab 13.4. Individual state versions can be managed using the GitLab REST API. -Users with the [Developer role](../../permissions.md) can retrieve state versions using their serial number. To retrieve a version: +If you have at least the Developer role, you can retrieve state versions by using their serial number:: ```shell curl --header "Private-Token: <your_access_token>" "https://gitlab.example.com/api/v4/projects/<your_project_id>/terraform/state/<your_state_name>/versions/<version-serial>" ``` -Users with the [Maintainer role](../../permissions.md) can remove state versions using their serial number. To remove a version: +If you have at least the Maintainer role, you can remove state versions by using their serial number: ```shell curl --header "Private-Token: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/<your_project_id>/terraform/state/<your_state_name>/versions/<version-serial>" ``` -## Remove a state file - -Users with at least the Maintainer role can use the -following options to remove a state file: +### Remove a state file -- **GitLab UI**: Go to **Infrastructure > Terraform**. In the **Actions** column, - click the vertical ellipsis (**{ellipsis_v}**) button and select - **Remove state file and versions**. -- **GitLab REST API**: You can remove a state file by making a request to the - REST API. For example: +If you have at least the Maintainer role, you can remove a state file. - ```shell - curl --header "Private-Token: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/<your_project_id>/terraform/state/<your_state_name>" - ``` - -- [GitLab GraphQL API](#remove-a-state-file-with-the-gitlab-graphql-api). - -### Remove a state file with the GitLab GraphQL API - -You can remove a state file by making a GraphQL API request. For example: +1. On the left sidebar, select **Infrastructure > Terraform**. +1. In the **Actions** column, select **Actions** (**{ellipsis_v}**) and then **Remove state file and versions**. -```shell -mutation deleteState { - terraformStateDelete(input: { id: "<global_id_for_the_state>" }) { - errors - } -} -``` +### Remove a state file by using the API -You can obtain the `<global_id_for_the_state>` by querying the list of states: +You can remove a state file by making a request to the REST API. For example: ```shell -query ProjectTerraformStates { - project(fullPath: "<your_project_path>") { - terraformStates { - nodes { - id - name - } - } - } -} +curl --header "Private-Token: <your_access_token>" --request DELETE "https://gitlab.example.com/api/v4/projects/<your_project_id>/terraform/state/<your_state_name>" ``` -For those new to the GitLab GraphQL API, read -[Getting started with GitLab GraphQL API](../../../api/graphql/getting_started.md). +You can also use [the GraphQL API](../../../api/graphql/reference/index.md#mutationterraformstatedelete). ## Related topics - [Troubleshooting GitLab-managed Terraform state](troubleshooting.md). +- To use GitLab and Terraform to deploy an AWS EC2 instance in a custom VPC, + see [this sample project](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-aws). diff --git a/doc/user/infrastructure/iac/troubleshooting.md b/doc/user/infrastructure/iac/troubleshooting.md index bc0aa39bc70..9dfe58396e2 100644 --- a/doc/user/infrastructure/iac/troubleshooting.md +++ b/doc/user/infrastructure/iac/troubleshooting.md @@ -38,33 +38,37 @@ To workaround this issue, make sure to apply one of the following conditions: 1. Grant Maintainer or Owner role to the `terraform-user` user on `subgroup-B`. 1. The `terraform-user` inherited access to `subgroup-B` and `subgroup-B` contains at least one project. -### Invalid CI/CD syntax error when using the `latest` base template +### Invalid CI/CD syntax error when using the base template -On GitLab 14.2 and later, you might get a CI/CD syntax error when using the -`latest` Base Terraform template: +You might encounter a CI/CD syntax error when using the Terraform templates: + +- On GitLab 14.2 and later, using the `latest` template. +- On GitLab 15.0 and later, using any version of the template. + +For example: ```yaml include: + # On 14.2 and later, when using either of the following: - template: Terraform/Base.latest.gitlab-ci.yml + - template: Terraform.latest.gitlab-ci.yml + # On 15.0 and later, the following templates have also been updated: + - template: Terraform/Base.gitlab-ci.yml + - template: Terraform.gitlab-ci.yml -my-Terraform-job: - extends: .init +my-terraform-job: + extends: .apply ``` -The base template's [jobs were renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67719/) -with better Terraform-specific names. To resolve the syntax error, you can: - -- Use the stable `Terraform/Base.gitlab-ci.yml` template, which has not changed. -- Update your pipeline configuration to use the new job names in - `https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml`. - For example: +There are two different causes for the error: - ```yaml - include: - - template: Terraform/Base.latest.gitlab-ci.yml +- In the case of `.init`, the error occurs because the init stage and jobs [were removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71188) from the templates, since they are no longer required. To resolve the syntax error, you can safely remove any jobs extending `.init`. +- For all other jobs, the reason for the failure is that the base jobs have been [renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67719): A `.terraform:` prefix has been added to every job name. For example, `.apply` became `.terraform:apply`. To fix this error, you can update the base job names. For example: - my-Terraform-job: - extends: .terraform:init # The updated name. + ```diff + my-terraform-job: + - extends: .apply + + extends: .terraform:apply ``` ## Troubleshooting Terraform state @@ -80,7 +84,7 @@ This happens because the value of `$CI_JOB_TOKEN` is only valid for the duration As a workaround, use [http backend configuration variables](https://www.terraform.io/docs/language/settings/backends/http.html#configuration-variables) in your CI job, which is what happens behind the scenes when following the -[Get started using GitLab CI](terraform_state.md#get-started-using-gitlab-ci) instructions. +[Get started using GitLab CI](terraform_state.md#initialize-a-terraform-state-as-a-backend-by-using-gitlab-cicd) instructions. ### Error: "address": required field is not set diff --git a/doc/user/markdown.md b/doc/user/markdown.md index fc2f1de5ce2..bbcb0f62a5c 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -8,9 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > The abbreviation [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/24592) from `GFM` to `GLFM` in GitLab 14.10. -GitLab automatically renders Markdown content. For example, when you add a comment to an issue, -you type the text in the Markdown language. When you save the issue, the text is rendered -with a set of styles. These styles are described on this page. +When you enter text in the GitLab UI, GitLab assumes the text is in the Markdown language. +The text is rendered with a set of styles. These styles are called *GitLab Flavored Markdown*. For example, in Markdown, an unordered list looks like this: diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 59bb4a89b0b..fe7a41aa542 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -626,10 +626,10 @@ Use your own URLs to complete the following steps: docker pull gitlab.example.com/org/build/sample_project/cr:v2.9.1 ``` -NOTE: -For container registry authentication, use either a -[personal access token](../../profile/personal_access_tokens.md) or a -[deploy token](../../project/deploy_tokens/index.md). + NOTE: + For container registry authentication, use either a + [personal access token](../../profile/personal_access_tokens.md) or a + [deploy token](../../project/deploy_tokens/index.md). 1. Rename the images to match the new project name: @@ -695,7 +695,7 @@ There may be some errors not properly cached. Follow these steps to investigate `200 OK`, the body might have the error `AccessDenied`. This indicates a permission problem from the S3 side. -1. Ensure your S3 configuration has the `deleteObject` permisson scope. Here's an +1. Ensure your S3 configuration has the `deleteObject` permission scope. Here's an [example role for an S3 bucket](../../../administration/object_storage.md#iam-permissions). Once adjusted, trigger another tag deletion. You should be able to successfully delete tags. diff --git a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md index 5f678a661f8..9dcb4d7127d 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md +++ b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md @@ -7,11 +7,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Reduce Container Registry data transfers **(FREE)** Depending on the frequency with which images or tags are downloaded from the Container Registry, -data transfers can exceed the GitLab limit. This page offers several recommendations and tips for +data transfers can exceed the GitLab.com limit. This page offers several recommendations and tips for reducing the amount of data you transfer with the Container Registry. -are downloaded from the Container Registry, data transfers can exceed the GitLab limit. This page -offers several recommendations and tips for reducing the amount of data you transfer with the -Container Registry. ## Check data transfer use @@ -115,9 +112,15 @@ images can be specified as a cache source by using multiple `--cache-from` argum up your builds and reduce the amount of data transferred. For more information, see the [documentation on Docker layer caching](../../../ci/docker/using_docker_build.md#make-docker-in-docker-builds-faster-with-docker-layer-caching). +## Check automation frequency + +We often create automation scripts bundled into container images to perform regular tasks on specific intervals. +You can reduce the frequency of those intervals in cases where the automation is pulling container images from +the GitLab Registry to a service outside of GitLab.com. + ## Move to GitLab Premium or Ultimate -GitLab data transfer limits are set at the tier level. If you need a higher limit, consider +GitLab.com data transfer limits are set at the tier level. If you need a higher limit, consider upgrading to [GitLab Premium or Ultimate](https://about.gitlab.com/upgrade/). ## Purchase additional data transfer diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 2cfe99876fa..7b52a6a8ce3 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -24,6 +24,7 @@ which doesn't include the Container Registry. To track work on this, see the epi > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15398) in GitLab 12.8. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/218737) from "expiration policy" to "cleanup policy" in GitLab 13.2. +> - [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. The cleanup policy is a scheduled job you can use to remove tags from the Container Registry. For the project where it's defined, tags matching the regex pattern are removed. @@ -158,12 +159,8 @@ Here are examples of regex patterns you may want to use: ### Set cleanup limits to conserve resources > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/288812) in GitLab 13.9 [with a flag](../../../administration/feature_flags.md) named `container_registry_expiration_policies_throttling`. Disabled by default. -> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80815) in GitLab 14.9. - -FLAG: -By default this feature is available in GitLab 14.9. To disable the feature, an administrator can -[disable the feature flag](../../../administration/feature_flags.md) -named `container_registry_expiration_policies_throttling`. +> - [Enabled by default](https://gitlab.com/groups/gitlab-org/-/epics/2270) in GitLab 14.9. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84996) the feature flag `container_registry_expiration_policies_throttling` in GitLab 15.0. Cleanup policies are executed as a background process. This process is complex, and depending on the number of tags to delete, the process can take time to finish. diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index 5e66c8ed7a5..af54d928bec 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -37,7 +37,8 @@ For a list of planned additions, view the To enable or turn off the Dependency Proxy for a group: -1. Go to your group's **Settings > Packages & Registries**. +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Packages & Registries**. 1. Expand the **Dependency Proxy** section. 1. To enable the proxy, turn on **Enable Proxy**. To turn it off, turn the toggle off. @@ -49,7 +50,8 @@ for the entire GitLab instance. To view the Dependency Proxy: -- Go to your group's **Packages & Registries > Dependency Proxy**. +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Packages & Registries > Dependency Proxy**. The Dependency Proxy is not available for projects. @@ -63,17 +65,8 @@ Prerequisites: ### Authenticate with the Dependency Proxy -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7. -> - It's [deployed behind a feature flag](../../feature_flags.md), enabled by default. -> - It's enabled on GitLab.com. -> - It's recommended for production use. -> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](../../../administration/packages/dependency_proxy.md#disabling-authentication). **(FREE SELF)** - -WARNING: -This feature might not be available to you. Check the **version history** note above for details. -The requirement to authenticate is a breaking change added in 13.7. An [administrator can temporarily -disable it](../../../administration/packages/dependency_proxy.md#disabling-authentication) if it -has disrupted your existing Dependency Proxy usage. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 [with a flag](../../../administration/feature_flags.md) named `dependency_proxy_for_private_groups`. Enabled by default. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/276777) the feature flag `dependency_proxy_for_private_groups` in GitLab 15.0. Because the Dependency Proxy is storing Docker images in a space associated with your group, you must authenticate against the Dependency Proxy. @@ -182,8 +175,9 @@ You can also use [custom CI/CD variables](../../../ci/variables/index.md#custom- To store a Docker image in Dependency Proxy storage: -1. Go to your group's **Packages & Registries > Dependency Proxy**. -1. Copy the **Dependency Proxy URL**. +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Packages & Registries > Dependency Proxy**. +1. Copy the **Dependency Proxy image prefix**. 1. Use one of these commands. In these examples, the image is `alpine:latest`. 1. You can also pull images by digest to specify exactly which version of an image to pull. diff --git a/doc/user/packages/dependency_proxy/reduce_dependency_proxy_storage.md b/doc/user/packages/dependency_proxy/reduce_dependency_proxy_storage.md index cd04d2e696b..839684da875 100644 --- a/doc/user/packages/dependency_proxy/reduce_dependency_proxy_storage.md +++ b/doc/user/packages/dependency_proxy/reduce_dependency_proxy_storage.md @@ -26,6 +26,8 @@ image or tag from Docker Hub. ## Cleanup policies +> [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. + ### Enable cleanup policies from within GitLab > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340777) in GitLab 14.6 diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md index 9dc859a37e2..37e1f0c3eb1 100644 --- a/doc/user/packages/generic_packages/index.md +++ b/doc/user/packages/generic_packages/index.md @@ -101,7 +101,8 @@ API or the UI. #### Do not allow duplicate Generic packages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/293755) in GitLab 13.12. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/293755) in GitLab 13.12. +> - [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. To prevent users from publishing duplicate generic packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index 29455fdbb35..d2edfcb94c5 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -106,6 +106,11 @@ the scope set to `api` or `read_api`. Open your [`~/.netrc`](https://everything.curl.dev/usingcurl/netrc) file and add the following text. Replace the variables in `< >` with your values. +WARNING: +If you use an environment variable called `NETRC`, Go will use its value +as a filename and ignore `~/.netrc`. If you intend to use `~/.netrc` in +the GitLab CI **do not use `NETRC` as an environment variable name**. + ```plaintext machine <url> login <username> password <token> ``` diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 6a515b78fc1..1e3b5bdc323 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -184,7 +184,7 @@ published to the GitLab Package Registry. Project name (default: test): ``` -1. Enter a project name or press Enter to use the directory name as project name. +1. Enter a project name or press <kbd>Enter</kbd> to use the directory name as project name. ## Authenticate to the Package Registry with Maven @@ -617,7 +617,8 @@ To delete these older package versions, consider using the Packages API or the U #### Do not allow duplicate Maven packages -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296895) in GitLab 13.9. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296895) in GitLab 13.9. +> - [Required permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) changed from developer to maintainer in GitLab 15.0. To prevent users from publishing duplicate Maven packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index 4d46032d229..eee6d55a3ce 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -325,7 +325,8 @@ python -m twine upload --repository <source_name> dist/<package_file> ### Publishing packages with the same name or version You cannot publish a package if a package of the same name and version already exists. -You must delete the existing package first. If you attempt to publish the same package +You must [delete the existing package](../../packages/package_registry/reduce_package_registry_storage.md#delete-a-package) first. +If you attempt to publish the same package more than once, a `400 Bad Request` error occurs. ## Install a PyPI package diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 2282a7d876e..d28cf75d11f 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -67,13 +67,12 @@ The following table lists project permissions available for each role: | [Application security](application_security/index.md):<br>Create and run [on-demand DAST scans](application_security/dast/index.md#on-demand-scans) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):<br>Manage [security policy](application_security/policies/index.md) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):<br>View [dependency list](application_security/dependency_list/index.md) | | | ✓ | ✓ | ✓ | -| [Application security](application_security/index.md):<br>View [threats list](application_security/threat_monitoring/index.md#threat-monitoring) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):<br>Create a [CVE ID Request](application_security/cve_id_request.md) | | | | ✓ | ✓ | | [Application security](application_security/index.md):<br>Create or assign [security policy project](application_security/policies/index.md) | | | | | ✓ | | [Clusters](infrastructure/clusters/index.md):<br>View [pod logs](project/clusters/kubernetes_pod_logs.md) | | | ✓ | ✓ | ✓ | | [Clusters](infrastructure/clusters/index.md):<br>View clusters | | | ✓ | ✓ | ✓ | | [Clusters](infrastructure/clusters/index.md):<br>Manage clusters | | | | ✓ | ✓ | -| [Container Registry](packages/container_registry/index.md):<br>Create, edit, delete [cleanup policies](packages/container_registry/index.md#delete-images-by-using-a-cleanup-policy) | | | ✓ | ✓ | ✓ | +| [Container Registry](packages/container_registry/index.md):<br>Create, edit, delete [cleanup policies](packages/container_registry/index.md#delete-images-by-using-a-cleanup-policy) | | | | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):<br>Push an image to the Container Registry | | | ✓ | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):<br>Pull an image from the Container Registry | ✓ (*20*) | ✓ (*20*) | ✓ | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):<br>Remove a Container Registry image | | | ✓ | ✓ | ✓ | @@ -144,7 +143,7 @@ The following table lists project permissions available for each role: | [Projects](project/index.md):<br>Create [snippets](snippets.md) | | ✓ | ✓ | ✓ | ✓ | | [Projects](project/index.md):<br>Manage labels | | ✓ | ✓ | ✓ | ✓ | | [Projects](project/index.md):<br>View [project traffic statistics](../api/project_statistics.md) | | ✓ | ✓ | ✓ | ✓ | -| [Projects](project/index.md):<br>Create, edit, delete [milestones](project/milestones/index.md). | | | ✓ | ✓ | ✓ | +| [Projects](project/index.md):<br>Create, edit, delete [milestones](project/milestones/index.md). | | ✓ | ✓ | ✓ | ✓ | | [Projects](project/index.md):<br>Create, edit, delete [releases](project/releases/index.md) | | | ✓ (*12*) | ✓ (*12*) | ✓ (*12*) | | [Projects](project/index.md):<br>Create, edit [wiki](project/wiki/index.md) pages | | | ✓ | ✓ | ✓ | | [Projects](project/index.md):<br>Enable [Review Apps](../ci/review_apps/index.md) | | | ✓ | ✓ | ✓ | @@ -391,6 +390,7 @@ The following table lists group permissions available for each role: | Publish [packages](packages/index.md) | | | ✓ | ✓ | ✓ | | Pull [packages](packages/index.md) | | ✓ | ✓ | ✓ | ✓ | | Delete [packages](packages/index.md) | | | | ✓ | ✓ | +| Create/edit/delete [Maven and generic package duplicate settings](packages/generic_packages/index.md#do-not-allow-duplicate-generic-packages) | | | | ✓ | ✓ | | Pull a Container Registry image | ✓ (7) | ✓ | ✓ | ✓ | ✓ | | Remove a Container Registry image | | | ✓ | ✓ | ✓ | | View [Group DevOps Adoption](group/devops_adoption/index.md) | | ✓ | ✓ | ✓ | ✓ | @@ -398,11 +398,12 @@ The following table lists group permissions available for each role: | View [Productivity analytics](analytics/productivity_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | Create and edit [group wiki](project/wiki/group.md) pages | | | ✓ | ✓ | ✓ | | Create project in group | | | ✓ (3)(5) | ✓ (3) | ✓ (3) | -| Create/edit/delete group milestones | | | ✓ | ✓ | ✓ | -| Create/edit/delete iterations | | | ✓ | ✓ | ✓ | +| Create/edit/delete group milestones | | ✓ | ✓ | ✓ | ✓ | +| Create/edit/delete iterations | | ✓ | ✓ | ✓ | ✓ | | Create/edit/delete metrics dashboard annotations | | | ✓ | ✓ | ✓ | -| Enable/disable a dependency proxy | | | ✓ | ✓ | ✓ | +| Enable/disable a dependency proxy | | | | ✓ | ✓ | | Purge the dependency proxy for a group | | | | | ✓ | +| Create/edit/delete dependency proxy [cleanup policies](packages/dependency_proxy/reduce_dependency_proxy_storage.md#cleanup-policies) | | | | ✓ | ✓ | | Use [security dashboard](application_security/security_dashboard/index.md) | | | ✓ | ✓ | ✓ | | View group Audit Events | | | ✓ (7) | ✓ (7) | ✓ | | Create subgroup | | | | ✓ (1) | ✓ | @@ -445,7 +446,7 @@ The following table lists group permissions available for each role: ### Subgroup permissions When you add a member to a subgroup, they inherit the membership and -permission level from the parent group(s). This model allows access to +permission level from the parent groups. This model allows access to nested groups if you have membership in one of its parents. To learn more, read through the documentation on @@ -549,7 +550,7 @@ Auditor users are given read-only access to all projects, groups, and other resources on the GitLab instance. An Auditor user should be able to access all projects and groups of a GitLab instance -with the permissions described on the documentation on [auditor users permissions](../administration/auditor_users.md#permissions-and-restrictions-of-an-auditor-user). +with the permissions described on the documentation on [auditor users permissions](../administration/auditor_users.md#auditor-user-permissions-and-restrictions). [Read more about Auditor users.](../administration/auditor_users.md) diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index a86968654c7..d7eb5040416 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -381,6 +381,34 @@ Each of these contribution events is tracked: - Design - Wiki page +### Retrieve user activity as a feed + +GitLab provides RSS feeds of user activity. To subscribe to the +RSS feed of a user's activity: + +1. Go to the [user's profile](#access-your-user-profile). +1. In the top right, select the feed symbol **{rss}** to display the results as an RSS feed in Atom format. + +The URL of the result contains both a feed token, and +the user's activity that you're authorized to view. +You can add this URL to your feed reader. + +### Reset the user activity feed token + +Feed tokens are sensitive and can reveal information from confidential issues. +If you think your feed token has been exposed, you should reset it. + +To reset your feed token: + +1. On the top bar, in the top right corner, select your avatar. +1. Select **Edit profile**. +1. On the left sidebar, select **Access Tokens**. +1. Scroll down. In the **Feed token** section, select the + **reset this token** link. +1. On the confirmation box, select **OK**. + +A new token is generated. + ## Troubleshooting ### Why do I keep getting signed out? diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index d0e9b427f1c..3eda363d108 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -7,6 +7,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Notification emails **(FREE)** +> Enhanced email styling [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78604) in GitLab 14.9 [with a feature flag](../../administration/feature_flags.md) named `enhanced_notify_css`. Disabled by default. +> Enhanced email styling [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 14.9. +> Enhanced email styling [enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 15.0. + Stay informed about what's happening in GitLab with email notifications. You can receive updates about activity in issues, merge requests, epics, and designs. @@ -24,7 +28,7 @@ You might receive notifications for one of the following reasons: or edit, or someone mentions <sup>1</sup> you. - You've [enabled notifications in an issue, merge request, or epic](#notifications-on-issues-merge-requests-and-epics). - You've configured notifications for the [project](#change-level-of-project-notifications) or [group](#group-notifications). -- You're subscribed to group or project pipeline notifications via the pipeline emails [integration](../project/integrations/overview.md). +- You're subscribed to group or project pipeline notifications via the pipeline emails [integration](../project/integrations/index.md). 1. GitLab doesn't send a notification when [a comment is edited to include a user mention](../discussions/index.md#editing-a-comment-to-add-a-mention). diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 4c132094d24..09e71ce9133 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -23,7 +23,7 @@ Personal access tokens are: - Required when [two-factor authentication (2FA)](account/two_factor_authentication.md) is enabled. - Used with a GitLab username to authenticate with GitLab features that require usernames. For example, - [GitLab managed Terraform state backend](../infrastructure/iac/terraform_state.md#using-a-gitlab-managed-terraform-state-backend-as-a-remote-data-source) + [GitLab-managed Terraform state backend](../infrastructure/iac/terraform_state.md#use-your-gitlab-backend-as-a-remote-data-source) and [Docker container registry](../packages/container_registry/index.md#authenticate-with-the-container-registry), - Similar to [project access tokens](../project/settings/project_access_tokens.md) and [group access tokens](../group/settings/group_access_tokens.md), but are attached to a user rather than a project or group. @@ -109,9 +109,7 @@ Personal access tokens expire on the date you define, at midnight UTC. - GitLab runs a check at 01:00 AM UTC every day to identify personal access tokens that expire in the next seven days. The owners of these tokens are notified by email. - GitLab runs a check at 02:00 AM UTC every day to identify personal access tokens that expire on the current date. The owners of these tokens are notified by email. - In GitLab Ultimate, administrators can - [limit the lifetime of personal access tokens](../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens). -- In GitLab Ultimate, administrators can choose whether or not to - [enforce personal access token expiration](../admin_area/settings/account_and_limit_settings.md#allow-expired-personal-access-tokens-to-be-used-deprecated). + [limit the lifetime of access tokens](../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens). ## Create a personal access token programmatically **(FREE SELF)** diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index ecd6e83efa1..c654eb2b0e8 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -91,8 +91,8 @@ Introduced in GitLab 13.6, the themes [Solarized](https://gitlab.com/gitlab-org/ ## Diff colors -A diff compares the old/removed content with the new/added content (e.g. when -[reviewing a merge request](../project/merge_requests/reviews/index.md#review-a-merge-request) or in a +A diff compares the old/removed content with the new/added content (e.g. when +[reviewing a merge request](../project/merge_requests/reviews/index.md#review-a-merge-request) or in a [Markdown inline diff](../markdown.md#inline-diff)). Typically, the colors red and green are used for removed and added lines in diffs. The exact colors depend on the selected [syntax highlighting theme](#syntax-highlighting-theme). diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index 344caed7449..a76517a7341 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -25,9 +25,6 @@ If there is a problem with the new version of the application, only a small percentage of users are affected and the change can either be fixed or quickly reverted. -Leveraging [Kubernetes' Canary deployments](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/#canary-deployments), visualize your canary -deployments right inside the [deploy board](deploy_boards.md), without the need to leave GitLab. - ## Use cases Canary deployments can be used when you want to ship features to only a portion of @@ -45,38 +42,7 @@ may want to consider [setting `service.spec.sessionAffinity` to `ClientIP` in your Kubernetes service definitions](https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies), but that is beyond the scope of this document. -## Enabling Canary Deployments - -Canary deployments require that you properly configure deploy boards: - -1. Follow the steps to [enable deploy boards](deploy_boards.md#enabling-deploy-boards). -1. To track canary deployments you must label your Kubernetes deployments and - pods with `track: canary`. To get started quickly, you can use the [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy) - template for canary deployments that GitLab provides. - -Depending on the deploy, the label should be either `stable` or `canary`. -GitLab assumes the track label is `stable` if the label is blank or missing. -Any other track label is considered `canary` (temporary). -This allows GitLab to discover whether a deployment is stable or canary (temporary). - -Once all of the above are set up and the pipeline has run at least once, -Go to the environments page under **Pipelines > Environments**. -As the pipeline executes, deploy boards clearly mark canary pods, enabling -quick and clear insight into the status of each environment and deployment. - -Canary deployments are marked with a yellow dot in the deploy board so that you -can quickly notice them. - - - -### Advanced traffic control with Canary Ingress (DEPRECATED) - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215501) in GitLab 13.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212320) from GitLab Premium to GitLab Free in 13.8. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. - -WARNING: -This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +## Advanced traffic control with Canary Ingress Canary deployments can be more strategic with [Canary Ingress](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary), which is an advanced traffic routing service that controls incoming HTTP @@ -84,7 +50,7 @@ requests between stable and canary deployments based on factors such as weight, and others. GitLab uses this service in its [Auto Deploy architecture](../../topics/autodevops/upgrading_auto_deploy_dependencies.md#v2-chart-resource-architecture) to let users quickly and safely roll out their new deployments. -#### How to set up a Canary Ingress in a canary deployment +### How to set up a Canary Ingress in a canary deployment A Canary Ingress is installed by default if your Auto DevOps pipeline uses [`v2.0.0+` of `auto-deploy-image`](../../topics/autodevops/upgrading_auto_deploy_dependencies.md#verify-dependency-versions). @@ -106,14 +72,51 @@ Here's an example setup flow from scratch: 1. [Run a new Auto DevOps pipeline](../../ci/pipelines/index.md#run-a-pipeline-manually) and make sure that the `canary` job succeeds and creates a canary deployment with Canary Ingress. -#### How to check the current traffic weight on a Canary Ingress +### Show Canary Ingress deployments on deploy boards (deprecated) + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215501) in GitLab 13.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212320) from GitLab Premium to GitLab Free in 13.8. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. + +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. + +To view canary deployments you must properly configure deploy boards: + +1. Follow the steps to [enable deploy boards](deploy_boards.md#enabling-deploy-boards). +1. To track canary deployments you must label your Kubernetes deployments and + pods with `track: canary`. To get started quickly, you can use the [Auto Deploy](../../topics/autodevops/stages.md#auto-deploy) + template for canary deployments that GitLab provides. + +Depending on the deploy, the label should be either `stable` or `canary`. +GitLab assumes the track label is `stable` if the label is blank or missing. +Any other track label is considered `canary` (temporary). +This allows GitLab to discover whether a deployment is stable or canary (temporary). + +Once all of the above are set up and the pipeline has run at least once, +Go to the environments page under **Pipelines > Environments**. +As the pipeline executes, deploy boards clearly mark canary pods, enabling +quick and clear insight into the status of each environment and deployment. + +Canary deployments are marked with a yellow dot in the deploy board so that you +can quickly notice them. + + + +#### How to check the current traffic weight on a Canary Ingress (deprecated) + +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. 1. Visit the [deploy board](../../user/project/deploy_boards.md). 1. View the current weights on the right.  -#### How to change the traffic weight on a Canary Ingress +#### How to change the traffic weight on a Canary Ingress (deprecated) + +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. You can change the traffic weight in your environment's deploy board by using [GraphiQL](../../api/graphql/getting_started.md#graphiql), or by sending requests to the [GraphQL API](../../api/graphql/getting_started.md#command-line). diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index 82106c9d1a9..935bc01bae7 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -226,7 +226,7 @@ on the running pod. If you are using a self-managed GitLab instance, you need to configure Amazon credentials. GitLab uses these credentials to assume an Amazon IAM role to create your cluster. -Create an IAM user and ensure it has permissions to assume the role(s) that +Create an IAM user and ensure it has permissions to assume the roles that your users need to create EKS clusters. For example, the following policy document allows assuming a role whose name starts with diff --git a/doc/user/project/clusters/add_existing_cluster.md b/doc/user/project/clusters/add_existing_cluster.md index f2d537513b7..c55c11151ce 100644 --- a/doc/user/project/clusters/add_existing_cluster.md +++ b/doc/user/project/clusters/add_existing_cluster.md @@ -27,7 +27,7 @@ To add any cluster to GitLab, you need: - Either a GitLab.com account or an account for a self-managed installation running GitLab 12.5 or later. - The Maintainer role for group-level and project-level clusters. -- Access to the Admin area for instance-level clusters. +- Access to the Admin Area for instance-level clusters. - A Kubernetes cluster. - Cluster administration access to the cluster with `kubectl`. @@ -230,3 +230,22 @@ kubectl create clusterrolebinding permissive-binding \ --user=kubelet \ --group=system:serviceaccounts ``` + +## Troubleshooting + +### `There was a problem authenticating with your cluster. Please ensure your CA Certificate and Token are valid` + +If you encounter this error while connecting a Kubernetes cluster, ensure you're +properly pasting the service token. Some shells may add a line break to the +service token, making it invalid. Ensure that there are no line breaks by +pasting your token into an editor and removing any additional spaces. + +You may also experience this error if your certificate is not valid. To check that your certificate's +subject alternative names contain the correct domain for your cluster's API, run this command: + +```shell +echo | openssl s_client -showcerts -connect kubernetes.example.com:443 2>/dev/null | +openssl x509 -inform pem -noout -text +``` + +The `-connect` argument expects a `host:port` combination. For example, `https://kubernetes.example.com` would be `kubernetes.example.com:443`. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index a4f6dc325c8..8cdd1792e7f 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -10,60 +10,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 14.0. -To create a new cluster use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac). - -NOTE: -Every new Google Cloud Platform (GCP) account receives -[$300 in credit upon sign up](https://console.cloud.google.com/freetrial). -In partnership with Google, GitLab is able to offer an additional $200 for new GCP -accounts to get started with the GitLab integration with Google Kubernetes Engine. -[Follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) -to apply for credit. - -NOTE: -Watch the webcast [Scalable app deployment with GitLab and Google Cloud Platform](https://about.gitlab.com/webcast/scalable-app-deploy/) -and learn how to spin up a Kubernetes cluster managed by Google Cloud Platform (GCP) -in a few clicks. - -## Create new cluster - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 14.0. - -As of GitLab 14.0, use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac) -to **safely create new clusters from GitLab**. - -Creating clusters from GitLab using cluster certificates is still available on the -GitLab UI but was **deprecated** in GitLab 14.0 and is scheduled for removal in -GitLab 15.0. We don't recommend using this method. - -You can create a new cluster hosted in EKS, GKE, on premises, and with other -providers using cluster certificates: - -- [New cluster hosted on Google Kubernetes Engine (GKE)](add_gke_clusters.md). -- [New cluster hosted on Amazon Elastic Kubernetes Service (EKS)](add_eks_clusters.md). - -To host them on premises and with other providers, you can use Terraform -or your preferred tool of choice to create and connect a cluster with GitLab. -The [GitLab Terraform provider](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs/resources/project_cluster) -supports connecting existing clusters using the certificate-based connection method. - -## Add existing cluster - -As of GitLab 14.0, use the [GitLab agent](../../clusters/agent/index.md) -to connect your cluster to GitLab. - -Alternatively, you can [add an existing cluster](add_existing_cluster.md) -through the certificate-based method, but we don't recommend using this method for [security implications](../../infrastructure/clusters/connect/index.md#security-implications-for-clusters-connected-with-certificates). - -## Configure your cluster - -As of GitLab 14.0, use the [GitLab agent](../../clusters/agent/index.md) -to configure your cluster. +To create and manage a new cluster use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac). ## Disable a cluster -When you successfully create a new Kubernetes cluster or add an existing -one to GitLab, the cluster connection to GitLab becomes enabled. To disable it: +When you successfully connect an existing cluster using cluster certificates, the cluster connection to GitLab becomes enabled. To disable it: 1. Go to your: - Project's **{cloud-gear}** **Infrastructure > Kubernetes clusters** page, for a project-level cluster. @@ -95,26 +46,3 @@ To remove the Kubernetes cluster integration: 1. Go to your cluster details page. 1. Select the **Advanced Settings** tab. 1. Select either **Remove integration** or **Remove integration and resources**. - -## Access controls - -See [cluster access controls (RBAC or ABAC)](cluster_access.md). - -## Troubleshooting - -### There was a problem authenticating with your cluster. Please ensure your CA Certificate and Token are valid - -If you encounter this error while adding a Kubernetes cluster, ensure you're -properly pasting the service token. Some shells may add a line break to the -service token, making it invalid. Ensure that there are no line breaks by -pasting your token into an editor and removing any additional spaces. - -You may also experience this error if your certificate is not valid. To check that your certificate's -subject alternative names contain the correct domain for your cluster's API, run this: - -```shell -echo | openssl s_client -showcerts -connect kubernetes.example.com:443 2>/dev/null | -openssl x509 -inform pem -noout -text -``` - -Note that the `-connect` argument expects a `host:port` combination. For example, `https://kubernetes.example.com` would be `kubernetes.example.com:443`. diff --git a/doc/user/project/clusters/cluster_access.md b/doc/user/project/clusters/cluster_access.md index 8ff0a275649..43ceb3673d8 100644 --- a/doc/user/project/clusters/cluster_access.md +++ b/doc/user/project/clusters/cluster_access.md @@ -28,7 +28,7 @@ Helm also creates additional service accounts and other resources for each installed application. Consult the documentation of the Helm charts for each application for details. -If you are [adding an existing Kubernetes cluster](add_remove_clusters.md#add-existing-cluster), +If you are [adding an existing Kubernetes cluster](add_existing_cluster.md), ensure the token of the account has administrator privileges for the cluster. The resources created by GitLab differ depending on the type of cluster. diff --git a/doc/user/project/clusters/deploy_to_cluster.md b/doc/user/project/clusters/deploy_to_cluster.md index c1cdf754c11..fc41533b17c 100644 --- a/doc/user/project/clusters/deploy_to_cluster.md +++ b/doc/user/project/clusters/deploy_to_cluster.md @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. To connect your cluster to GitLab, use the [GitLab agent](../../clusters/agent/index.md). -To deploy with the agent, use the [CI/CD workflow](../../clusters/agent/ci_cd_tunnel.md). +To deploy with the agent, use the [CI/CD workflow](../../clusters/agent/ci_cd_workflow.md). A Kubernetes cluster can be the destination for a deployment job. If diff --git a/doc/user/project/clusters/gitlab_managed_clusters.md b/doc/user/project/clusters/gitlab_managed_clusters.md index 9c5cc14f720..e295abf8d31 100644 --- a/doc/user/project/clusters/gitlab_managed_clusters.md +++ b/doc/user/project/clusters/gitlab_managed_clusters.md @@ -9,15 +9,19 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 11.5. > - Became [optional](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26565) in GitLab 11.11. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. To connect your cluster to GitLab, use the [GitLab agent](../../../user/clusters/agent/index.md). To manage applications, use the [Cluster Project Management Template](../../../user/clusters/management_project_template.md). +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `certificate_based_clusters`. + You can choose to allow GitLab to manage your cluster for you. If your cluster is managed by GitLab, resources for your projects are automatically created. See -the [Access controls](add_remove_clusters.md#access-controls) section for +the [Access controls](cluster_access.md) section for details about the created resources. If you choose to manage your own cluster, project-specific resources aren't created diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index b5e2a1bad51..b1158be9fb6 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -4,14 +4,23 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Kubernetes Logs (DEPRECATED) **(FREE)** +# Kubernetes Logs (DEPRECATED) **(FREE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4752) in GitLab 11.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26383) from GitLab Ultimate to GitLab Free 12.9. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/360182) behind a [feature flag](../../../administration/feature_flags.md) named `monitor_logging` in GitLab 15.0. Disabled by default. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: +This feature is in its end-of-life process. This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +It will be [removed completely](https://gitlab.com/gitlab-org/gitlab/-/issues/346485) in GitLab 15.2. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `monitor_logging` and the one named `certificate_based_clusters`. +On GitLab.com, this feature is not available. +This feature is not recommended for production use. GitLab makes it easy to view the logs of running pods in [connected Kubernetes clusters](index.md). By displaying the logs directly in GitLab diff --git a/doc/user/project/clusters/protect/container_host_security/index.md b/doc/user/project/clusters/protect/container_host_security/index.md deleted file mode 100644 index c897100f14e..00000000000 --- a/doc/user/project/clusters/protect/container_host_security/index.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Container Host Security **(FREE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Container Host Security is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -Container Host Security in GitLab provides Intrusion Detection and Prevention capabilities that can -monitor and (optionally) block activity inside the containers themselves. This is done by leveraging -an integration with Falco to provide the monitoring capabilities and an integration with Pod -Security Policies and AppArmor to provide blocking capabilities. - -## Overview - -Container Host Security can be used to monitor and block activity inside a container as well as to -enforce security policies across the entire Kubernetes cluster. Falco profiles allow for users to -define the activity they want to monitor for and detect. Among other things, this can include system -log entries, process starts, file activity, and network ports opened. AppArmor is used to block any -undesired activity via AppArmor profiles. These profiles are loaded into the cluster when -referenced by Pod Security Policies. - -By default, Container Host Security is deployed into the cluster in monitor mode only, with no -default profiles or rules running out-of-the-box. Activity monitoring and blocking begins only when -users define profiles for these technologies. - -## Installation - -See the [installation guide](quick_start_guide.md) for the recommended steps to install the -Container Host Security capabilities. This guide shows the recommended way of installing Container -Host Security through the Cluster Management Project. However, it's also possible to do a manual -installation through our Helm chart. - -## Features - -- Prevent containers from starting as root. -- Limit the privileges and system calls available to containers. -- Monitor system logs, process starts, files read/written/deleted, and network ports opened. -- Optionally block processes from starting or files from being read/written/deleted. - -## Supported container orchestrators - -Kubernetes v1.14+ is the only supported container orchestrator. OpenShift and other container -orchestrators aren't supported. - -## Supported Kubernetes providers - -The following cloud providers are supported: - -- Amazon EKS -- Google GKE - -Although Container Host Security may function on Azure or self-managed Kubernetes instances, it isn't -officially tested and supported on those providers. - -## Roadmap - -See the [Category Direction page](https://about.gitlab.com/direction/protect/container_host_security/) -for more information on the product direction of Container Host Security. diff --git a/doc/user/project/clusters/protect/container_host_security/quick_start_guide.md b/doc/user/project/clusters/protect/container_host_security/quick_start_guide.md deleted file mode 100644 index af3128e3006..00000000000 --- a/doc/user/project/clusters/protect/container_host_security/quick_start_guide.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Getting started with Container Host Security **(FREE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Container Host Security is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The following steps are recommended for installing Container Host Security. - -## Installation steps - -The following steps are recommended to install and use Container Host Security through GitLab: - -1. [Install at least one runner and connect it to GitLab](https://docs.gitlab.com/runner/). -1. [Create a group](../../../../group/#create-a-group). -1. [Connect a Kubernetes cluster to the group](../../add_remove_clusters.md). -1. [Create a cluster management project and associate it with the Kubernetes cluster](../../../../clusters/management_project.md). - -1. Install and configure an Ingress node: - - - [Install the Ingress node via CI/CD (Cluster Management Project)](../../../../clusters/applications.md#install-ingress-using-gitlab-cicd). - - Navigate to the Kubernetes page and enter the [DNS address for the external endpoint](../../gitlab_managed_clusters.md#base-domain) - into the **Base domain** field on the **Details** tab. Save the changes to the Kubernetes - cluster. - -1. [Install and configure Falco](../../../../clusters/applications.md#install-falco-using-gitlab-cicd) - for activity monitoring. -1. [Install and configure AppArmor](../../../../clusters/applications.md#install-apparmor-using-gitlab-cicd) - for activity blocking. -1. [Configure Pod Security Policies](../../../../clusters/applications.md#using-podsecuritypolicy-in-your-deployments) - (required to be able to load AppArmor profiles). - -It's possible to install and manage Falco and AppArmor in other ways, such as installing them -manually in a Kubernetes cluster and then connecting it back to GitLab. These methods aren't -supported or documented. - -## Viewing the logs - -Falco logs can be viewed by running the following command in your Kubernetes cluster: - -```shell -kubectl -n gitlab-managed-apps logs -l app=falco -``` - -## Troubleshooting - -### Trouble connecting to the cluster - -Your CI/CD pipeline may occasionally fail or have trouble connecting to the cluster. Here are some -initial troubleshooting steps that resolve the most common problems: - -1. [Clear the cluster cache](../../gitlab_managed_clusters.md#clearing-the-cluster-cache) -1. If things still aren't working, a more assertive set of actions may help get things back to a - good state: - - - Stop and [delete the problematic environment](../../../../../ci/environments/#delete-a-stopped-environment) - in GitLab. - - Delete the relevant namespace in Kubernetes by running - `kubectl delete namespaces <insert-some-namespace-name>` in your Kubernetes cluster. - - Rerun the application project pipeline to redeploy the application. - -**Related documentation links:** - -- [Cluster Management Project](../../../../clusters/management_project.md) diff --git a/doc/user/project/clusters/protect/container_network_security/index.md b/doc/user/project/clusters/protect/container_network_security/index.md deleted file mode 100644 index b294859c660..00000000000 --- a/doc/user/project/clusters/protect/container_network_security/index.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Container Network Security **(FREE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Container Network Security is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -Container Network Security in GitLab provides basic firewall functionality by leveraging Cilium -NetworkPolicies to filter traffic going in and out of the cluster as well as traffic between pods -inside the cluster. Container Network Security can be used to enforce L3, L4, and L7 policies and -can prevent an attacker with control over one pod from spreading laterally to access other pods in -the same cluster. Both Ingress and Egress rules are supported. - -By default, Cilium is deployed in Detection-only mode and only logs attack attempts. GitLab provides -a set of out-of-the-box policies as examples and to help users get started. These policies are -disabled by default, as they must usually be customized to match application-specific needs. - -## Installation - -See the [installation guide](quick_start_guide.md) for the recommended steps to install GitLab -Container Network Security. This guide shows the recommended way of installing Container Network -Security through the Cluster Management Project. However, it's also possible to install Cilium -manually through our Helm chart. - -## Features - -- GitLab managed installation of Cilium. -- Support for L3, L4, and L7 policies. -- Ability to export logs to a SIEM. -- Statistics page showing volume of packets processed and dropped over time (Ultimate users only). -- Management of NetworkPolicies through code in a project (Available for auto DevOps users only). -- Management of CiliumNetworkPolicies through a UI policy manager (Ultimate users only). - -## Supported container orchestrators - -Kubernetes v1.14+ is the only supported container orchestrator. OpenShift and other container -orchestrators aren't supported. - -## Supported Kubernetes providers - -The following cloud providers are supported: - -- Amazon EKS -- Google GKE - -Although Container Network Security may function on Azure or self-managed Kubernetes instances, it -isn't officially tested and supported on those providers. - -## Supported NetworkPolicies - -GitLab only supports the use of CiliumNetworkPolicies. Although generic Kubernetes NetworkPolicies -or other kinds of NetworkPolicies may work, GitLab doesn't test or support them. - -## Managing NetworkPolicies through GitLab vs your cloud provider - -Some cloud providers offer integrations with Cilium or offer other ways to manage NetworkPolicies in -Kubernetes. GitLab Container Network Security doesn't support deployments that have NetworkPolicies -managed by an external provider. By choosing to manage NetworkPolicies through GitLab, you can take -advantage of the following benefits: - -- Support for handling NetworkPolicy infrastructure as code. -- Full revision history and audit log of all changes made. -- Ability to revert back to a previous version at any time. - -## Roadmap - -See the [Category Direction page](https://about.gitlab.com/direction/protect/container_network_security/) -for more information on the product direction of Container Network Security. diff --git a/doc/user/project/clusters/protect/container_network_security/quick_start_guide.md b/doc/user/project/clusters/protect/container_network_security/quick_start_guide.md deleted file mode 100644 index 7671ed7eb73..00000000000 --- a/doc/user/project/clusters/protect/container_network_security/quick_start_guide.md +++ /dev/null @@ -1,230 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Getting started with Container Network Security **(FREE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -Container Network Security is in its end-of-life process. It's [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -The following steps are recommended for installing Container Network Security. - -## Installation steps - -The following steps are recommended to install and use Container Network Security through GitLab: - -1. [Install at least one runner and connect it to GitLab](https://docs.gitlab.com/runner/). -1. [Create a group](../../../../group/#create-a-group). -1. [Connect a Kubernetes cluster to the group](../../add_remove_clusters.md). -1. [Create a cluster management project and associate it with the Kubernetes cluster](../../../../clusters/management_project.md). - -1. Install and configure an Ingress node: - - - [Install the Ingress node via CI/CD (Cluster Management Project)](../../../../clusters/applications.md#install-ingress-using-gitlab-cicd). - - Navigate to the Kubernetes page and enter the [DNS address for the external endpoint](../../gitlab_managed_clusters.md#base-domain) - into the **Base domain** field on the **Details** tab. Save the changes to the Kubernetes - cluster. - -1. [Install and configure Cilium](#use-the-cluster-management-template-to-install-cilium). -1. Be sure to restart all pods that were running before Cilium was installed by running this command - in your cluster: - - `kubectl get pods --all-namespaces -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,HOSTNETWORK:.spec.hostNetwork --no-headers=true | grep '<none>' | awk '{print "-n "$1" "$2}' | xargs -L 1 -r kubectl delete pod` - - You can skip this step if `nodeinit.restartPods` is set to `true` on your Helm chart. - -It's possible to install and manage Cilium in other ways. For example, you could use the GitLab Helm -chart to install Cilium manually in a Kubernetes cluster, and then connect it back to GitLab. -However, such methods aren't documented or officially supported by GitLab. - -### Use the Cluster Management template to install Cilium - -[Cilium](https://cilium.io/) is a networking plug-in for Kubernetes that you can use to implement -support for [`NetworkPolicy`](https://kubernetes.io/docs/concepts/services-networking/network-policies/) -resources. For more information, see [Network Policies](../../../../../topics/autodevops/stages.md#network-policy). - -You can use the [Cluster Management Project Template](../../../../clusters/management_project_template.md) -to install Cilium in your Kubernetes cluster. - -1. In your cluster management project, go to `helmfile.yaml` and uncomment `- path: applications/cilium/helmfile.yaml`. -1. In `applications/cilium/helmfile.yaml`, set `clusterType` to either `gke` or `eks` based on which Kubernetes provider your are using. - - ```yaml - environments: - default: - values: - # Set to "gke" or "eks" based on your cluster type - - clusterType: "" - ``` - -1. Merge or push these changes to the default branch of your cluster management project, -and [GitLab CI/CD](../../../../../ci/index.md) will automatically install Cilium. - -WARNING: -Installation and removal of the Cilium requires a **manual** -[restart](https://docs.cilium.io/en/stable/gettingstarted/k8s-install-helm/#restart-unmanaged-pods) -of all affected pods in all namespaces to ensure that they are -[managed](https://docs.cilium.io/en/stable/operations/troubleshooting/#ensure-managed-pod) -by the correct networking plug-in. When Hubble is enabled, its related pod might require a -restart depending on whether it started prior to Cilium. For more information, see -[Failed Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#failed-deployment) -in the Kubernetes docs. - -NOTE: -Major upgrades might require additional setup steps. For more information, see -the official [upgrade guide](https://docs.cilium.io/en/stable/operations/upgrade/). - -Support for installing the Cilium application is provided by the -GitLab Container Security group. If you run into unknown issues, -[open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new), and ping at -least 2 people from the -[Container Security group](https://about.gitlab.com/handbook/product/categories/#container-security-group). - -### Configure the Cilium Helm chart - -You can customize Cilium's Helm variables by editing the `applications/cilium/values.yaml` -file in your cluster management project. Refer to the [Cilium Helm reference](https://docs.cilium.io/en/stable/helm-reference/) -for the available configuration options. - -By default, Cilium's -[audit mode](https://docs.cilium.io/en/stable/gettingstarted/policy-creation/#enable-policy-audit-mode) -is enabled. In audit mode, Cilium doesn't drop disallowed packets. You -can use `policy-verdict` log to observe policy-related decisions. You -can disable audit mode by setting `policyAuditMode: false` in -`applications/cilium/values.yaml`. - -The Cilium monitor log for traffic is logged out by the -`cilium-monitor` sidecar container. You can check these logs with the following command: - -```shell -kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor -``` - -You can disable the monitor log in `application/cilium/values.yaml`: - -```yaml -monitor: - enabled: false -``` - -The [Hubble](https://github.com/cilium/hubble) monitoring daemon is enabled by default -and it's set to collect per namespace flow metrics. This metrics are accessible on the -[Threat Monitoring](../../../../application_security/threat_monitoring/index.md) -dashboard. You can disable Hubble by adding the following to -`applications/cilium/values.yaml`: - -```yaml -hubble: - enabled: false -``` - -You can also adjust Helm values for Hubble by using -`applications/cilium/values.yaml`: - -```yaml -hubble: - enabled: true - metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' -``` - -## Managing Network Policies - -Managing NetworkPolicies through GitLab is advantageous over managing the policies in Kubernetes -directly. Kubernetes doesn't provide a GUI editor, a change control process, or a revision history. -Network Policies can be managed through GitLab in one of two ways: - -- Management through a YAML file in each application's project (for projects using Auto DevOps). For - more information, see the [Network Policy documentation](../../../../../topics/autodevops/stages.md#network-policy). -- Management through the GitLab Policy management UI (for projects not using Auto DevOps). For more - information, see the [Container Network Policy documentation](../../../../application_security/policies/index.md#container-network-policy) (Ultimate only). - -Each method has benefits and drawbacks: - -| | YAML method | UI method (Ultimate only) | -|--|:------------|:-------------------------------| -| **Benefits** | A change control process is possible by requiring [MR Approvals](../../../merge_requests/approvals/index.md). All changes are fully tracked and audited in the same way that Git tracks the history of any file in its repository. | The UI provides a simple rules editor for users who are less familiar with the YAML syntax of NetworkPolicies. This view is a live representation of the policies currently deployed in the Kubernetes cluster. The UI also allows for multiple network policies to be created per environment. | -| **Drawbacks** | Only one network policy can be deployed per environment (although that policy can be as detailed as needed). Also, if changes were made in Kubernetes directly rather than through the `auto-deploy-values.yaml` file, the YAML file's contents don't represent the actual state of policies deployed in Kubernetes. | Policy changes aren't audited and a change control process isn't available. | - -Users are encouraged to choose one of the two methods to manage their policies. If users attempt to -use both methods simultaneously, when the application project pipeline runs the contents of the -NetworkPolicy in the `auto-deploy-values.yaml` file may override policies configured in the UI -editor. - -## Monitoring throughput **(ULTIMATE)** - -To view statistics for Container Network Security, you must follow the installation steps above and -configure GitLab integration with Prometheus. Also, if you use custom Helm values for Cilium, you -must enable Hubble with flow metrics for each namespace by adding the following lines to -your [Cilium values](#use-the-cluster-management-template-to-install-cilium): - -```yaml -hubble: - enabled: true - metrics: - enabled: - - 'flow:sourceContext=namespace;destinationContext=namespace' -``` - -Additional information about the statistics page is available in the -[documentation that describes the Threat Management UI](../../../../application_security/policies/index.md#container-network-policy). - -## Forwarding logs to a SIEM - -Cilium logs can be forwarded to a SIEM or an external logging system through syslog protocol by -installing and configuring Fluentd. Fluentd can be installed through the GitLab -[Cluster Management Project](../../../../clusters/applications.md#install-fluentd-using-gitlab-cicd). - -## Viewing the logs - -Cilium logs can be viewed by running the following command in your Kubernetes cluster: - -```shell -kubectl -n gitlab-managed-apps logs -l k8s-app=cilium -c cilium-monitor -``` - -## Troubleshooting - -### Traffic is not being blocked as expected - -By default, Cilium is installed in Audit mode only, meaning that NetworkPolicies log policy -violations but don't block any traffic. To set Cilium to Blocking mode, you must add the following -lines to the `applications/cilium/values.yaml` file in your cluster management project: - -```yaml -policyEnforcementMode: "always" - -monitor: - eventTypes: ["drop", "policy-verdict"] -``` - -### Traffic is not being allowed as expected - -Keep in mind that when Cilium is set to blocking mode (rather than Audit mode), NetworkPolicies -operate on an allow-list basis. If one or more NetworkPolicies apply to a node, then all traffic -that doesn't match at least one Policy is blocked. To resolve, add NetworkPolicies defining the -traffic that you want to allow in the node. - -### Trouble connecting to the cluster - -Occasionally, your CI/CD pipeline may fail or have trouble connecting to the cluster. Here are some -initial troubleshooting steps that resolve the most common problems: - -1. [Clear the cluster cache](../../gitlab_managed_clusters.md#clearing-the-cluster-cache). -1. If things still aren't working, a more assertive set of actions may help get things back into a - good state: - - - Stop and [delete the problematic environment](../../../../../ci/environments/index.md#delete-a-stopped-environment) in GitLab. - - Delete the relevant namespace in Kubernetes by running `kubectl delete namespaces <insert-some-namespace-name>` in your Kubernetes cluster. - - Rerun the application project pipeline to redeploy the application. - -**Related documentation links:** - -- [Cluster Management Project](../../../../clusters/management_project.md) diff --git a/doc/user/project/clusters/protect/index.md b/doc/user/project/clusters/protect/index.md deleted file mode 100644 index 6b89f7f1557..00000000000 --- a/doc/user/project/clusters/protect/index.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Protecting your deployed applications **(FREE)** - -> [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) in GitLab 15.0. - -WARNING: -The Container Network Security and Container Host Security features are in their end-of-life -processes. They're -[deprecated](https://gitlab.com/groups/gitlab-org/-/epics/7476) -in GitLab 14.8, and planned for [removal](https://gitlab.com/groups/gitlab-org/-/epics/7477) -in GitLab 15.0. - -GitLab makes it straightforward to protect applications deployed in [connected Kubernetes clusters](index.md). -These protections are available in the Kubernetes network layer and in the container itself. At -the network layer, the Container Network Security capabilities in GitLab provide basic firewall -functionality by leveraging Cilium NetworkPolicies to filter traffic going in and out of the cluster -and traffic between pods inside the cluster. Inside the container, Container Host Security provides -Intrusion Detection and Prevention capabilities that can monitor and block activity inside the -containers themselves. - -## Capabilities - -The following capabilities are available to protect deployed applications in Kubernetes: - -- Container Network Security - - [Overview](container_network_security/index.md) - - [Installation guide](container_network_security/quick_start_guide.md) -- Container Host Security - - [Overview](container_host_security/index.md) - - [Installation guide](container_host_security/quick_start_guide.md) diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index 9d623518f72..086e1fccf6c 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -40,8 +40,8 @@ for an overview of how this is accomplished in GitLab! To create an executable runbook, you need: - **Kubernetes** - A Kubernetes cluster is required to deploy the rest of the - applications. The simplest way to get started is to add a cluster using one - of the [GitLab integrations](../add_remove_clusters.md#create-new-cluster). + applications. The simplest way to get started is to connect a cluster using the + [GitLab agent](../../../clusters/agent/index.md). - **Ingress** - Ingress can provide load balancing, SSL termination, and name-based virtual hosting. It acts as a web proxy for your applications. - **JupyterHub** - [JupyterHub](https://jupyterhub.readthedocs.io/) is a multi-user diff --git a/doc/user/project/clusters/serverless/aws.md b/doc/user/project/clusters/serverless/aws.md index cf571abbf8a..93bc41dc24c 100644 --- a/doc/user/project/clusters/serverless/aws.md +++ b/doc/user/project/clusters/serverless/aws.md @@ -2,506 +2,10 @@ stage: Configure group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: '../../../../update/removals.md#gitlab-serverless' --- -# Deploying AWS Lambda function using GitLab CI/CD **(FREE)** +# Deploying AWS Lambda function using GitLab CI/CD (removed) **(FREE)** -GitLab allows users to easily deploy AWS Lambda functions and create rich serverless applications. - -GitLab supports deployment of AWS Lambda functions through GitLab CI/CD using the following Serverless frameworks: - -- [Serverless Framework with AWS](#serverless-framework) -- [AWS' Serverless Application Model (SAM)](#aws-serverless-application-model) - -## Serverless Framework - -The [Serverless Framework can deploy to AWS](https://www.serverless.com/framework/docs/providers/aws/). - -We have prepared an example with a step-by-step guide to create a simple function and deploy it on AWS. - -Additionally, in the [How To section](#how-to), you can read about different use cases like: - -- Running a function locally. -- Working with secrets. -- Setting up CORS. - -Alternatively, you can quickly [create a new project with a template](../../working_with_projects.md#create-a-project). The [`Serverless Framework/JS` template](https://gitlab.com/gitlab-org/project-templates/serverless-framework/) already includes all parts described below. - -### Example - -This example shows you how to: - -1. Create a basic AWS Lambda Node.js function. -1. Link the function to an API Gateway `GET` endpoint. - -#### Steps - -The example consists of the following steps: - -1. Creating a Lambda handler function. -1. Creating a `serverless.yml` file. -1. Crafting the `.gitlab-ci.yml` file. -1. Setting up your AWS credentials with your GitLab account. -1. Deploying your function. -1. Testing the deployed function. - -Lets take it step by step. - -#### Creating a Lambda handler function - -Your Lambda function is the primary handler of requests. In this case, create a very simple Node.js `hello` function: - -```javascript -'use strict'; - -module.exports.hello = async event => { - return { - statusCode: 200, - body: JSON.stringify( - { - message: 'Your function executed successfully!' - }, - null, - 2 - ), - }; -}; -``` - -Place this code in the file `src/handler.js`. - -`src` is the standard location for serverless functions, but is customizable should you desire that. - -In our case, `module.exports.hello` defines the `hello` handler to reference later in the `serverless.yml`. - -You can learn more about the [AWS Lambda Node.js function handler](https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html) and all its various options in its documentation. - -#### Creating a `serverless.yml` file - -In the root of your project, create a `serverless.yml` file containing configuration specifics for the Serverless Framework. - -Put the following code in the file: - -```yaml -service: gitlab-example -provider: - name: aws - runtime: nodejs14.x - -functions: - hello: - handler: src/handler.hello - events: - - http: GET hello -``` - -Our function contains a handler and a event. - -The handler definition provisions the Lambda function using the source code located `src/handler.hello`. - -The `events` declaration creates an AWS API Gateway `GET` endpoint to receive external requests and hand them over to the Lambda function via a service integration. - -You can read more about the [available properties and additional configuration possibilities](https://www.serverless.com/framework/docs/providers/aws/guide/serverless.yml/) of the Serverless Framework. - -#### Crafting the `.gitlab-ci.yml` file - -In a `.gitlab-ci.yml` file in the root of your project, place the following code: - -```yaml -image: node:latest - -stages: - - deploy - -production: - stage: deploy - before_script: - - npm config set prefix /usr/local - - npm install -g serverless - script: - - serverless deploy --stage production --verbose - environment: production -``` - -This example code does the following: - -1. Uses the `node:latest` image for all GitLab CI/CD builds -1. The `deploy` stage: - - Installs the Serverless Framework. - - Deploys the serverless function to your AWS account using the AWS credentials - defined above. - -#### Setting up your AWS credentials with your GitLab account - -In order to interact with your AWS account, the GitLab CI/CD pipelines require both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to be defined in your GitLab settings under **Settings > CI/CD > Variables**. -For more information please see [Create a custom variable in the UI](../../../../ci/variables/index.md#custom-variables-validated-by-gitlab). - - The AWS credentials you provide must include IAM policies that provision correct - access control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. - -#### Deploying your function - -`git push` the changes to your GitLab repository and the GitLab build pipeline deploys your function. - -Your GitLab deploy stage log contains output containing your AWS Lambda endpoint URL, -with log lines similar to this: - -```plaintext -endpoints: - GET - https://u768nzby1j.execute-api.us-east-1.amazonaws.com/production/hello -``` - -#### Manually testing your function - -Running the following `curl` command should trigger your function. -Your URL should be the one retrieved from the GitLab deploy stage log: - -```shell -curl "https://u768nzby1j.execute-api.us-east-1.amazonaws.com/production/hello" -``` - -That should output: - -```json -{ - "message": "Your function executed successfully!" -} -``` - -Hooray! You now have a AWS Lambda function deployed via GitLab CI/CD. - -Nice work! - -### How To - -In this section, we show you how to build on the basic example to: - -- Run the function locally. -- Set up secret variables. -- Set up CORS. - -#### Running function locally - -The `serverless-offline` plugin allows to run your code locally. To run your code locally: - -1. Add the following to your `serverless.yml`: - - ```yaml - plugins: - - serverless-offline - ``` - -1. Start the service by running the following command: - - ```shell - serverless offline - ``` - -Running the following `curl` command should trigger your function. - -```shell -curl "http://localhost:3000/hello" -``` - -It should output: - -```json -{ - "message": "Your function executed successfully!" -} -``` - -#### Secret variables - -Secrets are injected into your functions using environment variables. - -By defining variables in the provider section of the `serverless.yml`, you add them to -the environment of the deployed function: - -```yaml -provider: - # Other configuration omitted - # ... - environment: - A_VARIABLE: ${env:A_VARIABLE} -``` - -From there, you can reference them in your functions as well. -Remember to add `A_VARIABLE` to your GitLab CI/CD variables under **Settings > CI/CD > Variables** to be picked up and deployed with your function. - -NOTE: -Anyone with access to the AWS environment may be able to see the values of those -variables persisted in the lambda definition. - -#### Setting up CORS - -If you want to set up a web page that makes calls to your function, like we have done in the [template](https://gitlab.com/gitlab-org/project-templates/serverless-framework/), you need to deal with the Cross-Origin Resource Sharing (CORS). - -The quick way to do that is to add the `cors: true` flag to the HTTP endpoint in your `serverless.yml`: - -```yaml -functions: - hello: - handler: src/handler.hello - events: - - http: # Rewrite this part to enable CORS - path: hello - method: get - cors: true # <-- CORS here -``` - -You also need to return CORS specific headers in your function response: - -```javascript -'use strict'; - -module.exports.hello = async event => { - return { - statusCode: 200, - headers: { - // Uncomment the line below if you need access to cookies or authentication - // 'Access-Control-Allow-Credentials': true, - 'Access-Control-Allow-Origin': '*' - }, - body: JSON.stringify( - { - message: 'Your function executed successfully!' - }, - null, - 2 - ), - }; -}; -``` - -For more information, see the [Your CORS and API Gateway survival guide](https://www.serverless.com/blog/cors-api-gateway-survival-guide/) -blog post written by the Serverless Framework team. - -#### Writing automated tests - -The [Serverless Framework](https://gitlab.com/gitlab-org/project-templates/serverless-framework/) -example project shows how to use Jest, Axios, and `serverless-offline` plugin to do -automated testing of both local and deployed serverless function. - -### Examples and template - -The example code is available: - -- As a [clonable repository](https://gitlab.com/gitlab-org/serverless/examples/serverless-framework-js). -- In a version with [tests and secret variables](https://gitlab.com/gitlab-org/project-templates/serverless-framework/). - -You can also use a [template](../../working_with_projects.md#create-a-project) -(based on the version with tests and secret variables) from within the GitLab UI (see -the `Serverless Framework/JS` template). - -## AWS Serverless Application Model - -AWS Serverless Application Model is an open source framework for building serverless -applications. It makes it easier to build and deploy serverless applications. For more -details, please take a look at AWS documentation on [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/). - -### Deploying AWS Lambda function using AWS SAM and GitLab CI/CD - -GitLab allows developers to build and deploy serverless applications using the combination of: - -- [AWS Serverless Application Model (AWS SAM)](https://aws.amazon.com/serverless/sam/). -- GitLab CI/CD. - -### Example - -This example shows you how to: - -- Install SAM CLI. -- Create a sample SAM application including a Lambda function and API Gateway. -- Build and deploy the application to your AWS account using GitLab CI/CD. - -### Steps - -The example consists of the following steps: - -1. Installing SAM CLI. -1. Creating an AWS SAM application using SAM CLI. -1. Crafting the `.gitlab-ci.yml` file. -1. Setting up your AWS credentials with your GitLab account. -1. Deploying your application. -1. Testing the deployed function. - -### Installing SAM CLI - -AWS SAM provides a CLI called AWS SAM CLI to make it easier to create and manage -applications. - -Some steps in this documentation use SAM CLI. Follow the instructions for -[installing SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) -to install and configure SAM CLI. - -If you use [AWS Cloud9](https://aws.amazon.com/cloud9/) as your integrated development -environment (IDE), the following are installed for you: - -- [AWS Command Line Interface](https://docs.aws.amazon.com/en_pv/cli/latest/userguide/cli-chap-install.html) -- [SAM CLI](https://docs.aws.amazon.com/en_pv/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) -- [Docker](https://docs.docker.com/install/) and necessary Docker images. - -### Creating an AWS SAM application using SAM CLI - -To create a new AWS SAM application: - -1. Create a new GitLab project. -1. `git clone` the project into your local environment. -1. Change to the newly cloned project and create a new SAM app using the following command: - - ```shell - sam init -r python3.8 -n gitlabpoc --app-template "hello-world" - ``` - -1. `git push` the application back to the GitLab project. - -This creates a SAM app named `gitlabpoc` using the default configuration, a single -Python 3.8 function invoked by an [Amazon API Gateway](https://aws.amazon.com/api-gateway/) -endpoint. To see additional runtimes supported by SAM and options for `sam init`, run: - -```shell -sam init -h -``` - -### Setting up your AWS credentials with your GitLab account - -In order to interact with your AWS account, the GitLab CI/CD pipelines require both -`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` to be set in the project's CI/CD variables. - -To set these: - -1. Navigate to the project's **Settings > CI/CD**. -1. Expand the **Variables** section and create entries for `AWS_ACCESS_KEY_ID` and - `AWS_SECRET_ACCESS_KEY`. -1. Mask the credentials so they do not show in logs using the **Masked** toggle. - -The AWS credentials you provide must include IAM policies that provision correct access -control to AWS Lambda, API Gateway, CloudFormation, and IAM resources. - -### Crafting the `.gitlab-ci.yml` file - -In a [`.gitlab-ci.yml`](../../../../ci/yaml/index.md) file in the root of your project, -add the following and replace `<S3_bucket_name>` with the name of the S3 bucket where you -want to store your package: - -```yaml -image: python:latest - -stages: - - deploy - -production: - stage: deploy - before_script: - - apt-get update - - apt-get install -y python3-pip - - pip3 install awscli --upgrade - - pip3 install aws-sam-cli --upgrade - script: - - sam build - - sam package --output-template-file packaged.yaml --s3-bucket <S3_bucket_name> - - sam deploy --template-file packaged.yaml --stack-name gitlabpoc --s3-bucket <S3_bucket_name> --capabilities CAPABILITY_IAM --region us-east-1 - environment: production -``` - -Let's examine the configuration file more closely: - -- `image` specifies the Docker image to use for this build. This is the latest Python - image since the sample application is written in Python. -- AWS CLI and AWS SAM CLI are installed in the `before_script` section. -- SAM build, package, and deploy commands are used to build, package, and deploy the - application. - -### Deploying your application - -Push changes to your GitLab repository and the GitLab build pipeline -deploys your application. If your: - -- Build and deploy are successful, [test your deployed application](#testing-the-deployed-application). -- Build fails, look at the build log to see why the build failed. Some common reasons - the build might fail are: - - - Incompatible versions of software. For example, Python runtime version might be - different from the Python on the build machine. Address this by installing the - required versions of the software. - - You may not be able to access your AWS account from GitLab. Check the CI/CD variables - you set up with AWS credentials. - - You may not have permission to deploy a serverless application. Make sure you - provide all required permissions to deploy a serverless application. - -### Testing the deployed application - -To test the application you deployed, please go to the build log and follow the following steps: - -1. Click on "Show complete raw" on the upper right-hand corner: - -  - -1. Look for HelloWorldApi – API Gateway endpoint similar to shown below: - -  - -1. Use curl to test the API. For example: - - ```shell - curl "https://py4rg7qtlg.execute-api.us-east-1.amazonaws.com/Prod/hello/" - ``` - -Output should be: - -```json -{"message": "hello world"} -``` - -### Testing Locally - -AWS SAM provides functionality to test your applications locally. You must have AWS SAM -CLI installed locally for you to test locally. - -First, test the function. - -SAM provides a default event in `events/event.json` that includes a message body of: - -```plaintext -{\"message\": \"hello world\"} -``` - -If you pass that event into the `HelloWorldFunction`, it should respond with the same -body. - -Invoke the function by running: - -```shell -sam local invoke HelloWorldFunction -e events/event.json -``` - -Output should be: - -```json -{"message": "hello world"} -``` - -After you confirm that Lambda function is working as expected, test the API Gateway -using following steps. - -Start the API locally by running: - -```shell -sam local start-api -``` - -SAM again launches a Docker container, this time with a mocked Amazon API Gateway -listening on `localhost:3000`. - -Call the `hello` API by running: - -```shell -curl "http://127.0.0.1:3000/hello" -``` - -Output again should be: - -```json -{"message": "hello world"} -``` +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/6) in GitLab 14.3 and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86267) in GitLab 15.0. diff --git a/doc/user/project/clusters/serverless/img/function-details-loaded_v14_0.png b/doc/user/project/clusters/serverless/img/function-details-loaded_v14_0.png Binary files differdeleted file mode 100644 index a19d236fc39..00000000000 --- a/doc/user/project/clusters/serverless/img/function-details-loaded_v14_0.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/function-endpoint.png b/doc/user/project/clusters/serverless/img/function-endpoint.png Binary files differdeleted file mode 100644 index a38fe2cb6c2..00000000000 --- a/doc/user/project/clusters/serverless/img/function-endpoint.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/function-execution.png b/doc/user/project/clusters/serverless/img/function-execution.png Binary files differdeleted file mode 100644 index f60dd277081..00000000000 --- a/doc/user/project/clusters/serverless/img/function-execution.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/function-list_v12_7.png b/doc/user/project/clusters/serverless/img/function-list_v12_7.png Binary files differdeleted file mode 100644 index f2a27ce7b0f..00000000000 --- a/doc/user/project/clusters/serverless/img/function-list_v12_7.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/sam-api-endpoint.png b/doc/user/project/clusters/serverless/img/sam-api-endpoint.png Binary files differdeleted file mode 100644 index 3407b2684fd..00000000000 --- a/doc/user/project/clusters/serverless/img/sam-api-endpoint.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/sam-complete-raw.png b/doc/user/project/clusters/serverless/img/sam-complete-raw.png Binary files differdeleted file mode 100644 index 1130cd29d56..00000000000 --- a/doc/user/project/clusters/serverless/img/sam-complete-raw.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/img/serverless-page_v14_0.png b/doc/user/project/clusters/serverless/img/serverless-page_v14_0.png Binary files differdeleted file mode 100644 index f88eb4bdcd2..00000000000 --- a/doc/user/project/clusters/serverless/img/serverless-page_v14_0.png +++ /dev/null diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 29164da307b..432caa8476f 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -2,818 +2,10 @@ stage: Configure group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-08-22' +redirect_to: '../../../../update/removals.md#gitlab-serverless' --- -# Serverless (DEPRECATED) **(FREE)** +# Serverless (removed) **(FREE)** -> - Introduced in GitLab 11.5. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/6) in GitLab 14.3. - -WARNING: -Serverless is currently in [alpha](../../../../policy/alpha-beta-support.md#alpha-features). - -## Overview - -Serverless architectures offer Operators and Developers the ability write highly scalable applications without provisioning a single server. - -GitLab supports several ways deploy Serverless applications in both Kubernetes Environments and also major cloud Function as a Service (FaaS) environments. - -Currently we support: - -- [Knative](#knative): Build Knative applications with Knative and `gitlabktl` on GKE and EKS. -- [AWS Lambda](aws.md): Create serverless applications via the Serverless Framework and GitLab CI/CD. - -## Knative - -Run serverless workloads on Kubernetes using [Knative](https://cloud.google.com/knative/). - -Knative extends Kubernetes to provide a set of middleware components that are useful to build -modern, source-centric, container-based applications. Knative brings some significant benefits out -of the box through its main components: - -- [Serving](https://github.com/knative/serving): Request-driven compute that can scale to zero. -- [Eventing](https://github.com/knative/eventing): Management and delivery of events. - -For more information on Knative, visit the [Knative docs repository](https://github.com/knative/docs). - -With GitLab Serverless, you can deploy both FaaS and serverless applications. - -## Prerequisites - -To run Knative on GitLab, you need: - -1. **Existing GitLab project:** You need a GitLab project to associate all resources. The simplest way to get started: - - If you are planning on [deploying functions](#deploying-functions), - clone the [functions example project](https://gitlab.com/knative-examples/functions) to get - started. - - If you are planning on [deploying a serverless application](#deploying-serverless-applications), - clone the sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app) to get - started. -1. **Kubernetes Cluster:** An RBAC-enabled Kubernetes cluster is required to deploy Knative. - The simplest way to get started is to add a cluster using the GitLab [GKE integration](../add_remove_clusters.md). - The set of minimum recommended cluster specifications to run Knative is 3 nodes, 6 vCPUs, and 22.50 GB memory. -1. **GitLab Runner:** A runner is required to run the CI jobs that deploy serverless - applications or functions onto your cluster. You can install GitLab Runner - onto the [existing Kubernetes cluster](https://docs.gitlab.com/runner/install/kubernetes.html). -1. **Domain Name:** Knative provides its own load balancer using Istio, and an - external IP address or hostname for all the applications served by Knative. Enter a - wildcard domain to serve your applications. Configure your DNS server to use the - external IP address or hostname for that domain. -1. **`.gitlab-ci.yml`:** GitLab uses [Kaniko](https://github.com/GoogleContainerTools/kaniko) - to build the application. We also use [GitLab Knative tool](https://gitlab.com/gitlab-org/gitlabktl) - CLI to simplify the deployment of services and functions to Knative. -1. **`serverless.yml`** (for [functions only](#deploying-functions)): When using serverless to deploy functions, the `serverless.yml` file - contains the information for all the functions being hosted in the repository as well as a reference - to the runtime being used. -1. **`Dockerfile`** (for [applications only](#deploying-serverless-applications)): Knative requires a - `Dockerfile` in order to build your applications. It should be included at the root of your - project's repository and expose port `8080`. `Dockerfile` is not require if you plan to build serverless functions - using our [runtimes](https://gitlab.com/gitlab-org/serverless/runtimes). -1. **Prometheus** (optional): The [Prometheus cluster integration](../../../clusters/integrations.md#prometheus-cluster-integration) - allows you to monitor the scale and traffic of your serverless function/application. -1. **Logging** (optional): Configuring logging allows you to view and search request logs for your serverless function/application. - See [Configuring logging](#configuring-logging) for more information. - -## Configuring Knative - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58941) in GitLab 12.0. - -1. Follow the steps to - [add a Kubernetes - cluster](../add_remove_clusters.md). - -1. Ensure GitLab can manage Knative: - - For a non-GitLab managed cluster, ensure that the service account for the token - provided can manage resources in the `serving.knative.dev` API group. - - For a GitLab managed cluster, if you added the cluster in [GitLab 12.1 or later](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30235), - then GitLab already has the required access and you can proceed to the next step. - - Otherwise, you need to manually grant the GitLab service account the ability to manage - resources in the `serving.knative.dev` API group. Since every GitLab service account - has the `edit` cluster role, the simplest way to do this is with an - [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) - adding rules to the default `edit` cluster role: First, save the following YAML as - `knative-serving-only-role.yaml`: - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: knative-serving-only-role - labels: - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rules: - - apiGroups: - - serving.knative.dev - resources: - - configurations - - configurationgenerations - - routes - - revisions - - revisionuids - - autoscalers - - services - verbs: - - get - - list - - create - - update - - delete - - patch - - watch - ``` - - Then run the following command: - - ```shell - kubectl apply -f knative-serving-only-role.yaml - ``` - - Alternatively, permissions can be granted on a per-service account basis - using `Role`s and `RoleBinding`s (see the [Kubernetes RBAC - documentation](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) - for more information). - -1. Follow the steps to deploy [functions](#deploying-functions) - or [serverless applications](#deploying-serverless-applications) onto your - cluster. - -1. **Optional:** For invocation metrics to show in GitLab, additional Istio metrics need to be configured in your cluster. For example, with Knative v0.9.0, you can use [this manifest](https://gitlab.com/gitlab-org/charts/knative/-/raw/v0.10.0/vendor/istio-metrics.yml). - -## Supported runtimes - -Serverless functions for GitLab can be run using: - -- [GitLab-managed](#gitlab-managed-runtimes) runtimes. -- [OpenFaaS](#openfaas-runtimes) runtimes. - -If a runtime is not available for the required programming language, consider deploying a -[serverless application](#deploying-serverless-applications). - -### GitLab-managed runtimes - -The following GitLab-managed [runtimes](https://gitlab.com/gitlab-org/serverless/runtimes) -are available: - -- `go` (proof of concept) -- `nodejs` -- `ruby` - -You must provide a `Dockerfile` to run serverless functions if no runtime is specified. - -### OpenFaaS runtimes - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29253) in GitLab 12.5. - -[OpenFaaS classic runtimes](https://github.com/openfaas/templates#templates-in-store) can be used with GitLab serverless. - -OpenFaas runtimes are available for the following languages: - -- C# -- Go -- NodeJS -- PHP -- Python -- Ruby - -Runtimes are specified using the pattern: `openfaas/classic/<template_name>`. The following -example shows how to define a function in `serverless.yml` using an OpenFaaS runtime: - -```yaml -hello: - source: ./hello - runtime: openfaas/classic/ruby - description: "Ruby function using OpenFaaS classic runtime" -``` - -`handler` is not needed for OpenFaaS functions. The location of the handler is defined -by the conventions of the runtime. - -See the [`ruby-openfaas-function`](https://gitlab.com/knative-examples/ruby-openfaas-function) -project for an example of a function using an OpenFaaS runtime. - -## Deploying functions - -> Introduced in GitLab 11.6. - -You can find and import all the files referenced in this doc in the -**[functions example project](https://gitlab.com/knative-examples/functions)**. - -Follow these steps to deploy a function using the Node.js runtime to your -Knative instance (you can skip these steps if you've cloned the example -project): - -1. Create a directory to house the function. In this example we will - create a directory called `echo` at the root of the project. - -1. Create the file to contain the function code. In this example, our file is called `echo.js` and is located inside the `echo` directory. If your project is: - - Public, continue to the next step. - - Private, you must [create a GitLab deploy token](../../deploy_tokens/index.md#creating-a-deploy-token) with `gitlab-deploy-token` as the name and the `read_registry` scope. - -1. `.gitlab-ci.yml`: this defines a pipeline used to deploy your functions. - It must be included at the root of your repository: - - ```yaml - include: - - template: Serverless.gitlab-ci.yml - - functions:build: - extends: .serverless:build:functions - environment: production - - functions:deploy: - extends: .serverless:deploy:functions - environment: production - ``` - - This `.gitlab-ci.yml` creates jobs that invoke some predefined commands to - build and deploy your functions to your cluster. - - `Serverless.gitlab-ci.yml` is a template that allows customization. - You can either import it with `include` parameter and use `extends` to - customize your jobs, or you can inline the entire template by choosing it - from **Apply a template** dropdown when editing the `.gitlab-ci.yml` file through - the user interface. - -1. `serverless.yml`: this file contains the metadata for your functions, - such as name, runtime, and environment. - - It must be included at the root of your repository. - The following is a sample `echo` function which shows the required structure - for the file. - - You can find the relevant files for this project in the [functions example project](https://gitlab.com/knative-examples/functions). - - ```yaml - service: functions - description: "GitLab Serverless functions using Knative" - - provider: - name: triggermesh - envs: - FOO: value - secrets: - - my-secrets - - functions: - echo-js: - handler: echo-js - source: ./echo-js - runtime: gitlab/runtimes/nodejs - description: "node.js runtime function" - envs: - MY_FUNCTION: echo-js - secrets: - - my-secrets - ``` - -Explanation of the fields used above: - -### `service` - -| Parameter | Description | -|-----------|-------------| -| `service` | Name for the Knative service which serves the function. | -| `description` | A short description of the `service`. | - -### `provider` - -| Parameter | Description | -|-----------|-------------| -| `name` | Indicates which provider is used to execute the `serverless.yml` file. In this case, the TriggerMesh middleware. | -| `envs` | Includes the environment variables to be passed as part of function execution for **all** functions in the file, where `FOO` is the variable name and `BAR` are the variable contents. You may replace this with your own variables. | -| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for **all** functions in the file. The secrets are expected in `INI` format. | - -### `functions` - -In the `serverless.yml` example above, the function name is `echo` and the -subsequent lines contain the function attributes. - -| Parameter | Description | -|-----------|-------------| -| `handler` | The function's name. | -| `source` | Directory with sources of a functions. | -| `runtime` (optional)| The runtime to be used to execute the function. This can be a runtime alias (see [Runtime aliases](#runtime-aliases)), or it can be a full URL to a custom runtime repository. When the runtime is not specified, we assume that `Dockerfile` is present in the function directory specified by `source`. | -| `description` | A short description of the function. | -| `envs` | Sets an environment variable for the specific function only. | -| `secrets` | Includes the contents of the Kubernetes secret as environment variables accessible to be passed as part of function execution for the specific function only. The secrets are expected in `INI` format. | - -### Deployment - -#### Runtime aliases - -The optional `runtime` parameter can refer to one of the following runtime aliases (also see [Supported runtimes](#supported-runtimes)): - -| Runtime alias | Maintained by | -|-------------|---------------| -| `gitlab/runtimes/go` | GitLab | -| `gitlab/runtimes/nodejs` | GitLab | -| `gitlab/runtimes/ruby` | GitLab | -| `openfaas/classic/csharp` | OpenFaaS | -| `openfaas/classic/go` | OpenFaaS | -| `openfaas/classic/node` | OpenFaaS | -| `openfaas/classic/php7` | OpenFaaS | -| `openfaas/classic/python` | OpenFaaS | -| `openfaas/classic/python3` | OpenFaaS | -| `openfaas/classic/ruby` | OpenFaaS | - -After the `.gitlab-ci.yml` template has been added and the `serverless.yml` file -has been created, pushing a commit to your project results in a CI pipeline -being executed which deploys each function as a Knative service. After the -deploy stage has finished, additional details for the function display -under **Infrastructure > Serverless platform**. - - - -This page contains all functions available for the project, the description for -accessing the function, and, if available, the function's runtime information. -The details are derived from the Knative installation inside each of the project's -Kubernetes cluster. Click on each function to obtain detailed scale and invocation data. - -The function details can be retrieved directly from Knative on the cluster: - -```shell -kubectl -n "$KUBE_NAMESPACE" get services.serving.knative.dev -``` - -The sample function can now be triggered from any HTTP client using a simple `POST` call: - - 1. Using curl (replace the URL on the last line with the URL of your application): - - ```shell - curl \ - --header "Content-Type: application/json" \ - --request POST \ - --data '{"GitLab":"FaaS"}' \ - "http://functions-echo.functions-1.functions.example.com/" - ``` - - 1. Using a web-based tool (such as Postman or Restlet) - -  - -### Secrets - -To access your Kubernetes secrets from within your function, the secrets should be created under the namespace of your serverless deployment and specified in your `serverless.yml` file as above. -You can create secrets in several ways. The following sections show some examples. - -#### CLI example - -```shell -kubectl create secret generic my-secrets -n "$KUBE_NAMESPACE" --from-literal MY_SECRET=imverysecure -``` - -#### Part of deployment job - -You can extend your `.gitlab-ci.yml` to create the secrets during deployment using the [CI/CD variables](../../../../ci/variables/index.md) -stored securely under your GitLab project. - -```yaml -deploy:function: - stage: deploy - environment: production - extends: .serverless:deploy:functions - before_script: - - kubectl create secret generic my-secret - --from-literal MY_SECRET="$GITLAB_SECRET_VARIABLE" - --namespace "$KUBE_NAMESPACE" - --dry-run -o yaml | kubectl apply -f - -``` - -### Running functions locally - -Running a function locally is a good way to quickly verify behavior during development. - -Running functions locally requires: - -- Go 1.12 or newer installed. -- Docker Engine installed and running. -- `gitlabktl` installed using the Go package manager: - - ```shell - GO111MODULE=on go get gitlab.com/gitlab-org/gitlabktl - ``` - -To run a function locally: - -1. Navigate to the root of your GitLab serverless project. -1. Build your function into a Docker image: - - ```shell - gitlabktl serverless build - ``` - -1. Run your function in Docker: - - ```shell - docker run -itp 8080:8080 <your_function_name> - ``` - -1. Invoke your function: - - ```shell - curl "http://localhost:8080" - ``` - -## Deploying Serverless applications - -> Introduced in GitLab 11.5. - -Serverless applications are an alternative to [serverless functions](#deploying-functions). -They're useful in scenarios where an existing runtime does not meet the needs of -an application, such as one written in a language that has no runtime available. -Note though that serverless applications should be stateless. - -You can reference and import the sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app) -to get started. Add the following `.gitlab-ci.yml` to the root of your repository -(you may skip this step if you've previously cloned the previously mentioned, -sample [Knative Ruby App](https://gitlab.com/knative-examples/knative-ruby-app)): - -```yaml -include: - - template: Serverless.gitlab-ci.yml - -build: - extends: .serverless:build:image - -deploy: - extends: .serverless:deploy:image -``` - -`Serverless.gitlab-ci.yml` is a template that allows customization. -You can either import it with `include` parameter and use `extends` to -customize your jobs, or you can inline the entire template by choosing it -from **Apply a template** dropdown when editing the `.gitlab-ci.yml` file through -the user interface. - -A `serverless.yml` file is not required when deploying serverless applications. - -### Deploy the application with Knative - -With all the pieces in place, the next time a CI pipeline runs the Knative application deploys. Navigate to -**CI/CD > Pipelines** and click the most recent pipeline. - -### Function details - -Go to the **Infrastructure > Serverless platform** page to see the final URL of your functions. - - - -### Invocation metrics - -On the same page as above, click on one of the function -rows to bring up the function details page. - - - -The pod count gives you the number of pods running the serverless function instances on a given cluster. - -For the Knative function invocations to appear, the -[Prometheus cluster integration must be enabled](../../../clusters/integrations.md#prometheus-cluster-integration). - -Once Prometheus is enabled, a message may appear indicating that the metrics data _is -loading or is not available at this time._ It appears upon the first access of the -page, but should go away after a few seconds. If the message does not disappear, then it -is possible that GitLab is unable to connect to the Prometheus instance running on the -cluster. - -## Configuring logging - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33330) in GitLab 12.5. - -### Prerequisites - -- A GitLab-managed cluster. -- `kubectl` installed and working. - -Running `kubectl` commands on your cluster requires setting up access to the -cluster first. For clusters created on: - -- GKE, see [GKE Cluster Access](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) -- Other platforms, see [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/). - -### Enable request log template - -Run the following command to enable request logs: - -```shell -kubectl edit cm -n knative-serving config-observability -``` - -Copy the `logging.request-log-template` from the `data._example` field to the data field one level up in the hierarchy. - -### Enable request logs - -Run the following commands to install Elasticsearch, Kibana, and Filebeat into a `kube-logging` namespace and configure all nodes to forward logs using Filebeat: - -```shell -kubectl apply -f https://gitlab.com/gitlab-org/serverless/configurations/knative/raw/v0.7.0/kube-logging-filebeat.yaml -kubectl label nodes --all beta.kubernetes.io/filebeat-ready="true" -``` - -### Viewing request logs - -To view request logs: - -1. Run `kubectl proxy`. -1. Navigate to [Kibana UI](http://localhost:8001/api/v1/namespaces/kube-logging/services/kibana/proxy/app/kibana). - -Or: - -1. Open the [Kibana UI](http://localhost:8001/api/v1/namespaces/kube-logging/services/kibana/proxy/app/kibana). -1. Click on **Discover**, then select `filebeat-*` from the dropdown on the left. -1. Enter `kubernetes.container.name:"queue-proxy" AND message:/httpRequest/` into the search box. - -## Enabling TLS for Knative services - -By default, a GitLab serverless deployment is served over `http`. To serve -over `https`, you must manually obtain and install TLS certificates. - -The simplest way to accomplish this is to use Certbot to -[manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). -Certbot is a free, open source software tool for automatically using Let's Encrypt -certificates on manually-administered websites to enable HTTPS. - -The following instructions relate to installing and running Certbot on a Linux -server that has Python 3 installed, and may not work on other operating systems -or with other versions of Python. - -1. Install Certbot by running the - [`certbot-auto` wrapper script](https://eff-certbot.readthedocs.io/install.html#certbot-auto). - On the command line of your server, run the following commands: - - ```shell - wget https://dl.eff.org/certbot-auto - sudo mv certbot-auto /usr/local/bin/certbot-auto - sudo chown root /usr/local/bin/certbot-auto - sudo chmod 0755 /usr/local/bin/certbot-auto - /usr/local/bin/certbot-auto --help - ``` - - To check the integrity of the `certbot-auto` script, run: - - ```shell - wget -N https://dl.eff.org/certbot-auto.asc - gpg2 --keyserver ipv4.pool.sks-keyservers.net --recv-key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - gpg2 --trusted-key 4D17C995CD9775F2 --verify certbot-auto.asc /usr/local/bin/certbot-auto - ``` - - The output of the last command should look something like: - - ```shell - gpg: Signature made Mon 10 Jun 2019 06:24:40 PM EDT - gpg: using RSA key A2CFB51FA275A7286234E7B24D17C995CD9775F2 - gpg: key 4D17C995CD9775F2 marked as ultimately trusted - gpg: checking the trustdb - gpg: marginals needed: 3 completes needed: 1 trust model: pgp - gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u - gpg: next trustdb check due at 2027-11-22 - gpg: Good signature from "Let's Encrypt Client Team <letsencrypt-client@eff.org>" [ultimate] - ``` - -1. Run the following command to use Certbot to request a certificate - using DNS challenge during authorization: - - ```shell - /usr/local/bin/certbot-auto certonly --manual --preferred-challenges dns -d '*.<namespace>.example.com' - ``` - - Where `<namespace>` is the namespace created by GitLab for your serverless project (composed of `<project_name>-<project_id>-<environment>`) and - `example.com` is the domain being used for your project. If you are unsure what the namespace of your project is, navigate - to the **Infrastructure > Serverless platform** page of your project and inspect - the endpoint provided for your function/app. - -  - - In the above image, the namespace for the project is `node-function-11909507` and the domain is `knative.info`, thus - certificate request line would look like this: - - ```shell - ./certbot-auto certonly --manual --preferred-challenges dns -d '*.node-function-11909507.knative.info' - ``` - - The Certbot tool walks you through the steps of validating that you own each domain that you specify by creating TXT records in those domains. - After this process is complete, the output should look something like this: - - ```shell - IMPORTANT NOTES: - - Congratulations! Your certificate and chain have been saved at: - /etc/letsencrypt/live/namespace.example.com/fullchain.pem - Your key file has been saved at: - /etc/letsencrypt/live/namespace.example/privkey.pem - Your cert will expire on 2019-09-19. To obtain a new or tweaked - version of this certificate in the future, simply run certbot-auto - again. To non-interactively renew *all* of your certificates, run - "certbot-auto renew" - -----BEGIN PRIVATE KEY----- - - Your account credentials have been saved in your Certbot - configuration directory at /etc/letsencrypt. You should make a - secure backup of this folder now. This configuration directory will - also contain certificates and private keys obtained by Certbot so - making regular backups of this folder is ideal. - ``` - -1. Create certificate and private key files. Using the contents of the files - returned by Certbot, create two files in order to create the - Kubernetes secret: - - Run the following command to see the contents of `fullchain.pem`: - - ```shell - sudo cat /etc/letsencrypt/live/node-function-11909507.knative.info/fullchain.pem - ``` - - Output should look like this: - - ```shell - -----BEGIN CERTIFICATE----- - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b4ag== - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - K2fcb195768c39e9a94cec2c2e30Qg== - -----END CERTIFICATE----- - ``` - - Create a file with the name `cert.pem` with the contents of the entire output. - - Once `cert.pem` is created, run the following command to see the contents of `privkey.pem`: - - ```shell - sudo cat /etc/letsencrypt/live/namespace.example/privkey.pem - ``` - - Output should look like this: - - ```shell - -----BEGIN PRIVATE KEY----- - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - 2fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 04f294d1eaca42b8692017b426d53bbc8fe75f827734f0260710b83a556082df - -----BEGIN CERTIFICATE----- - fcb195768c39e9a94cec2c2e32c59c0aad7a3365c10892e8116b5d83d4096b6 - 4f294d1eaca42b8692017b4262== - -----END PRIVATE KEY----- - ``` - - Create a new file with the name `cert.pk` with the contents of the entire output. - -1. Create a Kubernetes secret to hold your TLS certificate, `cert.pem`, and - the private key `cert.pk`: - - NOTE: - Running `kubectl` commands on your cluster requires setting up access to the cluster first. - For clusters created on GKE, see - [GKE Cluster Access](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl). - For other platforms, [install `kubectl`](https://kubernetes.io/docs/tasks/tools/). - - ```shell - kubectl create --namespace istio-system secret tls istio-ingressgateway-certs \ - --key cert.pk \ - --cert cert.pem - ``` - - Where `cert.pem` and `cert.pk` are your certificate and private key files. Note that the `istio-ingressgateway-certs` secret name is required. - -1. Configure Knative to use the new secret that you created for HTTPS - connections. Run the - following command to open the Knative shared `gateway` in edit mode: - - ```shell - kubectl edit gateway knative-ingress-gateway --namespace knative-serving - ``` - - Update the gateway to include the following `tls:` section and configuration: - - ```shell - tls: - mode: SIMPLE - privateKey: /etc/istio/ingressgateway-certs/tls.key - serverCertificate: /etc/istio/ingressgateway-certs/tls.crt - ``` - - Example: - - ```shell - apiVersion: networking.istio.io/v1alpha3 - kind: Gateway - metadata: - # ... skipped ... - spec: - selector: - istio: ingressgateway - servers: - - hosts: - - "*" - port: - name: http - number: 80 - protocol: HTTP - - hosts: - - "*" - port: - name: https - number: 443 - protocol: HTTPS - tls: - mode: SIMPLE - privateKey: /etc/istio/ingressgateway-certs/tls.key - serverCertificate: /etc/istio/ingressgateway-certs/tls.crt - ``` - - After your changes are running on your Knative cluster, you can begin using the HTTPS protocol for secure access your deployed Knative services. - In the event a mistake is made during this process and you need to update the cert, you must edit the gateway `knative-ingress-gateway` - to switch back to `PASSTHROUGH` mode. Once corrections are made, edit the file again so the gateway uses the new certificates. - -## Using an older version of `gitlabktl` - -There may be situations where you want to run an older version of `gitlabktl`. This -requires setting an older version of the `gitlabktl` image in the `.gitlab-ci.yml` file. - -To set an older version, add `image:` to the `functions:deploy` block. For example: - -```yaml -functions:deploy: - extends: .serverless:deploy:functions - environment: production - image: registry.gitlab.com/gitlab-org/gitlabktl:0.5.0 -``` - -Different versions are available by changing the version tag at the end of the registry URL in the -format `registry.gitlab.com/gitlab-org/gitlabktl:<version>`. - -For a full inventory of available `gitlabktl` versions, see the `gitlabktl` project's -[container registry](https://gitlab.com/gitlab-org/gitlabktl/container_registry). +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/6) in GitLab 14.3 and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86267) in GitLab 15.0. diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 567c15c7d5f..42c1b8b0a62 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -16,12 +16,16 @@ type: howto, reference > This is [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60525) in > GitLab 13.12. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. [An epic exists](https://gitlab.com/groups/gitlab-org/-/epics/2493) to add this functionality to the [agent](../index.md). +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `certificate_based_clusters`. + GitLab deploy boards offer a consolidated view of the current health and status of each CI [environment](../../ci/environments/index.md) running on [Kubernetes](https://kubernetes.io), displaying the status of the pods in the deployment. Developers and other teammates can view the diff --git a/doc/user/project/import/cvs.md b/doc/user/project/import/cvs.md index 8bb716d8122..17d717bdc61 100644 --- a/doc/user/project/import/cvs.md +++ b/doc/user/project/import/cvs.md @@ -24,8 +24,8 @@ The following list illustrates the main differences between CVS and Git: whole, or they fail without any changes. In CVS, commits (and other operations) are not atomic. If an operation on the repository is interrupted in the middle, the repository can be left in an inconsistent state. -- **Storage method.** Changes in CVS are per file (changeset), while in Git - a committed file(s) is stored in its entirety (snapshot). That means it's +- **Storage method.** Changes in CVS are per file (changeset), while in Git, + committed files are stored in their entirety (snapshot). This means it is very easy in Git to revert or undo a whole change. - **Revision IDs.** The fact that in CVS changes are per files, the revision ID is depicted by version numbers, for example `1.4` reflects how many times a diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 329d91916e8..a190edb179d 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -7,66 +7,56 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Import your project from GitHub to GitLab **(FREE)** -Using the importer, you can import your GitHub repositories to GitLab.com or to -your self-managed GitLab instance. - -The following aspects of a project are imported: - -- Repository description (GitLab.com & 7.7+) -- Git repository data (GitLab.com & 7.7+) -- Issues (GitLab.com & 7.7+) -- Pull requests (GitLab.com & 8.4+) -- Wiki pages (GitLab.com & 8.4+) -- Milestones (GitLab.com & 8.7+) -- Labels (GitLab.com & 8.7+) -- Release note descriptions (GitLab.com & 8.12+) -- Pull request review comments (GitLab.com & 10.2+) -- Pull request reviews (GitLab.com & 13.7+) -- Pull request "merged by" information (GitLab.com & 13.7+) -- Regular issue and pull request comments -- [Git Large File Storage (LFS) Objects](../../../topics/git/lfs/index.md) -- Pull request comments replies in discussions ([GitLab.com & 14.5+](https://gitlab.com/gitlab-org/gitlab/-/issues/336596)) -- Diff Notes suggestions ([GitLab.com & 14.7+](https://gitlab.com/gitlab-org/gitlab/-/issues/340624)) - -References to pull requests and issues are preserved (GitLab.com & 8.7+), and -each imported repository maintains visibility level unless that [visibility -level is restricted](../../public_access.md#restrict-use-of-public-or-internal-projects), -in which case it defaults to the default project visibility. - -The namespace is a user or group in GitLab, such as `gitlab.com/janedoe` or -`gitlab.com/customer-success`. You can do some bulk actions to move projects to -different namespaces in the rails console. - -This process does not migrate or import any types of groups or organizations -from GitHub to GitLab. - -## Use cases - -The steps you take depend on whether you are importing from GitHub.com or -GitHub Enterprise. The steps also depend on whether you are importing to GitLab.com or -self-managed GitLab instance. - -- If you're importing to GitLab.com, you can alternatively import GitHub repositories - using a [personal access token](#use-a-github-token). We do not recommend - this method, as it does not associate all user activity (such as issues and - pull requests) with matching GitLab users. -- If you're importing to a self-managed GitLab instance, you can alternatively use the - [GitHub Rake task](../../../administration/raketasks/github_import.md) to import - projects without the constraints of a [Sidekiq](../../../development/sidekiq_style_guide.md) worker. -- If you're importing from GitHub Enterprise to your self-managed GitLab instance: - - You must first enable [GitHub integration](../../../integration/github.md). - - To import projects from GitHub Enterprise to GitLab.com, use the [Import API](../../../api/import.md). - - If GitLab is behind a HTTP/HTTPS proxy you must populate the [allowlist for local requests](../../../security/webhooks.md#allowlist-for-local-requests) - with `github.com` and `api.github.com` to solve the hostname. For more information, read the issue - [Importing a GitHub project requires DNS resolution even when behind a proxy](https://gitlab.com/gitlab-org/gitlab/-/issues/37941) -- If you're importing from GitHub.com to your self-managed GitLab instance, - setting up GitHub integration is not required. You can use the [Import API](../../../api/import.md). - -## How it works +You can import your GitHub repositories: + +- From either GitHub.com or GitHub Enterprise. +- To either GitLab.com or a self-managed GitLab instance. + +This process does not migrate or import any types of groups or organizations from GitHub to GitLab. + +The namespace is a user or group in GitLab, such as `gitlab.com/sidney-jones` or +`gitlab.com/customer-success`. You can use bulk actions in the rails console to move projects to +different namespaces. + +If you are importing to a self-managed GitLab instance, you can use the +[GitHub Rake task](../../../administration/raketasks/github_import.md) instead. This allows you to import projects +without the constraints of a [Sidekiq](../../../development/sidekiq/index.md) worker. + +If you are importing from GitHub Enterprise to a self-managed GitLab instance: + +- You must first enable [GitHub integration](../../../integration/github.md). +- To import projects from GitHub Enterprise to GitLab.com, use the [Import API](../../../api/import.md). +- If GitLab is behind a HTTP/HTTPS proxy, you must populate the [allowlist for local requests](../../../security/webhooks.md#allowlist-for-local-requests) + with `github.com` and `api.github.com` to solve the hostname. For more information, read the issue + [Importing a GitHub project requires DNS resolution even when behind a proxy](https://gitlab.com/gitlab-org/gitlab/-/issues/37941). + +If you are importing from GitHub.com to a self-managed GitLab instance: + +- Setting up GitHub integration is not required. +- You can use the [Import API](../../../api/import.md). + +When importing projects: + +- If a user referenced in the project is not found in the GitLab database, the project creator is set as the author and + assignee. The project creator is usually the user that initiated the import process. A note on the issue mentioning the + original GitHub author is added. +- The importer creates any new namespaces (or groups) if they do not exist, or, if the namespace is taken, the + repository is imported under the namespace of the user who initiated the import process. The namespace or repository + name can also be edited, with the proper permissions. +- The importer also imports branches on forks of projects related to open pull requests. These branches are + imported with a naming scheme similar to `GH-SHA-username/pull-request-number/fork-name/branch`. This may lead to + a discrepancy in branches compared to those of the GitHub repository. + +For additional technical details, refer to the [GitHub Importer](../../../development/github_importer.md) +developer documentation. + +For an overview of the import process, see the video [Migrating from GitHub to GitLab](https://youtu.be/VYOXuOg9tQI). + +## Prerequisites When issues and pull requests are being imported, the importer attempts to find -their GitHub authors and assignees in the database of the GitLab instance (note -that pull requests are called "merge requests" in GitLab). +their GitHub authors and assignees in the database of the GitLab instance. Pull requests are called _merge requests_ in +GitLab. For this association to succeed, each GitHub author and assignee in the repository must meet one of the following conditions prior to the import: @@ -75,32 +65,9 @@ must meet one of the following conditions prior to the import: - Have a GitHub account with a [public-facing email address](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-github-user-account/managing-email-preferences/setting-your-commit-email-address) that matches their GitLab account's email address. - NOTE: - GitLab content imports that use GitHub accounts require that the GitHub - public-facing email address is populated so that all comments and - contributions are properly mapped to the same user in GitLab. GitHub - Enterprise (on premise) does not require this field to be populated to use the - product, so you may have to add it on existing accounts for the imported - content to be properly mapped to the user in the new system. Refer to GitHub - documentation for instructions on how to add this email address. - -If a user referenced in the project is not found in the GitLab database, the project creator (typically the user -that initiated the import process) is set as the author/assignee, but a note on the issue mentioning the original -GitHub author is added. - -The importer creates any new namespaces (groups) if they do not exist, or, if the namespace is taken, the -repository is imported under the namespace of the user who initiated the import process. The namespace/repository -name can also be edited, with the proper permissions. - -The importer will also import branches on forks of projects related to open pull requests. These branches will be -imported with a naming scheme similar to `GH-SHA-username/pull-request-number/fork-name/branch`. This may lead to -a discrepancy in branches compared to those of the GitHub repository. - -For additional technical details, you can refer to the -[GitHub Importer](../../../development/github_importer.md "Working with the GitHub importer") -developer documentation. - -For an overview of the import process, see the video [Migrating from GitHub to GitLab](https://youtu.be/VYOXuOg9tQI). +GitLab content imports that use GitHub accounts require that the GitHub public-facing email address is populated. This means +all comments and contributions are properly mapped to the same user in GitLab. GitHub Enterprise does not require this +field to be populated so you may have to add it on existing accounts. ## Import your GitHub repository into GitLab @@ -108,10 +75,12 @@ For an overview of the import process, see the video [Migrating from GitHub to G Before you begin, ensure that any GitHub users who you want to map to GitLab users have either: -- A GitLab account that has logged in using the GitHub icon - \- or - +- A GitLab account that has logged in using the GitHub icon. - A GitLab account with an email address that matches the [publicly visible email address](https://docs.github.com/en/rest/reference/users#get-a-user) in the profile of the GitHub user +If you are importing to GitLab.com, you can alternatively import GitHub repositories using a [personal access token](#use-a-github-token). +We do not recommend this method, as it does not associate all user activity (such as issues and pull requests) with matching GitLab users. + User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with the user account that is performing the import. @@ -119,10 +88,10 @@ NOTE: If you are using a self-managed GitLab instance or if you are importing from GitHub Enterprise, this process requires that you have configured [GitHub integration](../../../integration/github.md). -1. From the top navigation bar, click **+** and select **New project**. +1. From the top navigation bar, select **+** and select **New project**. 1. Select the **Import project** tab and then select **GitHub**. 1. Select the first button to **List your GitHub repositories**. You are redirected to a page on [GitHub](https://github.com) to authorize the GitLab application. -1. Click **Authorize GitlabHQ**. You are redirected back to the GitLab Import page and all of your GitHub repositories are listed. +1. Select **Authorize GitlabHQ**. You are redirected back to the GitLab Import page and all of your GitHub repositories are listed. 1. Continue on to [selecting which repositories to import](#select-which-repositories-to-import). ### Use a GitHub token @@ -134,14 +103,13 @@ associate all user activity (such as issues and pull requests) with matching Git If you are an administrator of a self-managed GitLab instance or if you are importing from GitHub Enterprise, you cannot use a personal access token. The [GitHub integration method (above)](#use-the-github-integration) is recommended for all users. -Read more in the [How it works](#how-it-works) section. If you are not using the GitHub integration, you can still perform an authorization with GitHub to grant GitLab access your repositories: 1. Go to <https://github.com/settings/tokens/new> 1. Enter a token description. 1. Select the repository scope. -1. Click **Generate token**. +1. Select **Generate token**. 1. Copy the token hash. 1. Go back to GitLab and provide the token to the GitHub importer. 1. Hit the **List Your GitHub Repositories** button and wait while GitLab reads your repositories' information. @@ -161,7 +129,7 @@ your GitHub repositories are listed. you can filter projects by name. If filter is applied, **Import all repositories** only imports matched repositories. 1. The **Status** column shows the import status of each repository. You can choose to leave the page open and it will update in real-time or you can return to it later. -1. Once a repository has been imported, click its GitLab path to open its GitLab URL. +1. Once a repository has been imported, select its GitLab path to open its GitLab URL.  @@ -197,6 +165,30 @@ Reducing the time spent in cloning a repository can be done by increasing networ performance (by using high performance SSDs, for example) of the disks that store the Git repositories (for your GitLab instance). Increasing the number of Sidekiq workers does *not* reduce the time spent cloning repositories. +## Imported data + +The following items of a project are imported: + +- Repository description. +- Git repository data. +- Issues. +- Pull requests. +- Wiki pages. +- Milestones. +- Labels. +- Release note descriptions. +- Pull request review comments. +- Regular issue and pull request comments. +- [Git Large File Storage (LFS) Objects](../../../topics/git/lfs/index.md). +- Pull request reviews (GitLab.com and GitLab 13.7 and later). +- Pull request "merged by" information (GitLab.com and GitLab 13.7 and later). +- Pull request comments replies in discussions ([GitLab.com and GitLab 14.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/336596)). +- Diff Notes suggestions ([GitLab.com and GitLab 14.7 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/340624)). + +References to pull requests and issues are preserved. Each imported repository maintains visibility level unless that +[visibility level is restricted](../../public_access.md#restrict-use-of-public-or-internal-projects), in which case it +defaults to the default project visibility. + ## Alternative way to import notes and diff notes When GitHub Importer runs on extremely large projects not all notes & diff notes can be imported due to GitHub API `issues_comments` & `pull_requests_comments` endpoints limitation. diff --git a/doc/user/project/import/img/gitlab_import_history_page_v14_10.png b/doc/user/project/import/img/gitlab_import_history_page_v14_10.png Binary files differindex c93b5ed2b27..812696a8faa 100644 --- a/doc/user/project/import/img/gitlab_import_history_page_v14_10.png +++ b/doc/user/project/import/img/gitlab_import_history_page_v14_10.png diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index bf343078634..22e6d45dd96 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -46,7 +46,7 @@ integration in GitLab. 1. Select **Atlassian Bamboo**. 1. Ensure the **Active** checkbox is selected. 1. Enter the base URL of your Bamboo server. For example, `https://bamboo.example.com`. -1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](overview.md#ssl-verification). +1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](index.md#manage-ssl-verification). 1. Enter the [build key](#identify-the-bamboo-build-plan-build-key) from your Bamboo build plan. 1. If necessary, enter a username and password for a Bamboo user that has @@ -71,7 +71,7 @@ Bamboo. For example, `https://bamboo.example.com/browse/PROJ-PLAN`. ### Builds not triggered If builds are not triggered, ensure you entered the right GitLab IP address in -Bamboo under **Trigger IP addresses**. Also check [service hook logs](overview.md#troubleshooting-integrations) for request failures. +Bamboo under **Trigger IP addresses**. Also check [service hook logs](index.md#troubleshooting-integrations) for request failures. ### Advanced Atlassian Bamboo features not available in GitLab UI diff --git a/doc/user/project/integrations/bugzilla.md b/doc/user/project/integrations/bugzilla.md index 4a9a8d62098..0f7ce182e1a 100644 --- a/doc/user/project/integrations/bugzilla.md +++ b/doc/user/project/integrations/bugzilla.md @@ -57,4 +57,4 @@ internal issue tracker, the internal issue is linked. ## Troubleshooting -To see recent service hook deliveries, check [service hook logs](overview.md#troubleshooting-integrations). +To see recent service hook deliveries, check [service hook logs](index.md#troubleshooting-integrations). diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 2dae02dc093..dc56c2669f8 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w NOTE: The GitLab Slack application is only configurable for GitLab.com. It will **not** work for on-premises installations where you can configure the -[Slack slash commands](slack_slash_commands.md) service instead. We're planning +[Slack slash commands](slack_slash_commands.md) integration instead. We're planning to make this configurable for all GitLab installations, but there's no ETA - see [#28164](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). @@ -31,17 +31,21 @@ Alternatively, you can configure the Slack application with a project's integration settings. Keep in mind that you must have the appropriate permissions for your Slack -team to be able to install a new application, read more in Slack's +workspace to be able to install a new application. Read more in Slack's documentation on [Adding an app to your workspace](https://slack.com/help/articles/202035138-Add-apps-to-your-Slack-workspace). -To enable the GitLab service for your Slack team: +To enable the GitLab integration for your Slack workspace: 1. Go to your project's **Settings > Integration > Slack application** (only visible on GitLab.com). -1. Select **Add to Slack**. +1. Select **Install Slack app**. +1. Select **Allow** on Slack's confirmation screen. That's all! You can now start using the Slack slash commands. +You can also select **Reinstall Slack app** to update the app in your Slack workspace +to the latest version. See the [Version history](#version-history) for details. + ## Create a project alias for Slack To create a project alias on GitLab.com for Slack integration: @@ -62,7 +66,7 @@ GitLab error: project or alias not found ## Usage -After confirming the installation, you, and everyone else in your Slack team, +After confirming the installation, you, and everyone else in your Slack workspace, can use all the [slash commands](../../../integration/slash_commands.md). When you perform your first slash command, you are asked to authorize your @@ -78,3 +82,11 @@ project, you would do: ```plaintext /gitlab gitlab-org/gitlab issue show 1001 ``` + +## Version history + +### 15.0+ + +In GitLab 15.0 the Slack app is updated to [Slack's new granular permissions app model](https://medium.com/slack-developer-blog/more-precision-less-restrictions-a3550006f9c3). + +There is no change in functionality. A reinstall is not required but recommended. diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 9764c4d44a0..7af2e431157 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -6,26 +6,114 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Project integrations **(FREE)** -You can find the available integrations under your project's -**Settings > Integrations** page. You need to have at least -the Maintainer role on the project. +You can integrate your GitLab projects with other applications. Integrations are +like plugins, and give you the freedom to add +functionality to GitLab. -## Integrations +## View project integrations -Like plugins, integrations allow you to integrate GitLab with other applications, adding additional features. -For more information, read the -[overview of integrations](overview.md) or learn how to manage your integrations: +Prerequisites: -- *For GitLab 13.3 and later,* read [Project integration management](../../admin_area/settings/project_integration_management.md). -- *For GitLab 13.2 and earlier,* read [Integration Management](../../admin_area/settings/project_integration_management.md), - which replaced the deprecated Service Templates [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/268032) - in GitLab 14.0. +- You must have at least the Maintainer role for the project. -## Project webhooks +To view the available integrations for your project: -Project webhooks allow you to trigger a URL if for example new code is pushed or -a new issue is created. You can configure webhooks to listen for specific events -like pushes, issues or merge requests. GitLab sends a POST request with data -to the webhook URL. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. + +You can also view and manage integration settings across [all projects in an instance or group](../../admin_area/settings/project_integration_management.md). +For a single project, you can choose to inherit the instance or group configuration, +or provide custom settings. + +NOTE: +Instance and group-based integration management replaces service templates, which +were [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/268032) in GitLab 14.0. + +## Manage SSL verification + +By default, the SSL certificate for outgoing HTTP requests is verified based on +an internal list of Certificate Authorities. This means the certificate cannot +be self-signed. + +You can turn off SSL verification in the configuration settings for [webhooks](webhooks.md#configure-a-webhook-in-gitlab) +and some integrations. + +## Available integrations + +You can configure the following integrations. + +| Integration | Description | Integration hooks | +|-----------------------------------------------------------------------------|-----------------------------------------------------------------------|------------------------| +| [Asana](asana.md) | Add commit messages as comments to Asana tasks. | **{dotted-circle}** No | +| Assembla | Manage projects. | **{dotted-circle}** No | +| [Atlassian Bamboo CI](bamboo.md) | Run CI/CD pipelines with Atlassian Bamboo. | **{check-circle}** Yes | +| [Bugzilla](bugzilla.md) | Use Bugzilla as the issue tracker. | **{dotted-circle}** No | +| Buildkite | Run CI/CD pipelines with Buildkite. | **{check-circle}** Yes | +| Campfire | Connect to chat. | **{dotted-circle}** No | +| [Confluence Workspace](../../../api/integrations.md#confluence-integration) | Use Confluence Cloud Workspace as an internal wiki. | **{dotted-circle}** No | +| [Custom issue tracker](custom_issue_tracker.md) | Use a custom issue tracker. | **{dotted-circle}** No | +| [Datadog](../../../integration/datadog.md) | Trace your GitLab pipelines with Datadog. | **{check-circle}** Yes | +| [Discord Notifications](discord_notifications.md) | Send notifications about project events to a Discord channel. | **{dotted-circle}** No | +| Drone CI | Run CI/CD pipelines with Drone. | **{check-circle}** Yes | +| [Emails on push](emails_on_push.md) | Send commits and diff of each push by email. | **{dotted-circle}** No | +| [EWM](ewm.md) | Use IBM Engineering Workflow Management as the issue tracker. | **{dotted-circle}** No | +| [External wiki](../wiki/index.md#link-an-external-wiki) | Link an external wiki. | **{dotted-circle}** No | +| [Flowdock](../../../api/integrations.md#flowdock) | Send notifications from GitLab to Flowdock flows. | **{dotted-circle}** No | +| [GitHub](github.md) | Obtain statuses for commits and pull requests. | **{dotted-circle}** No | +| [Google Chat](hangouts_chat.md) | Send notifications from your GitLab project to a room in Google Chat. | **{dotted-circle}** No | +| [Harbor](harbor.md) | Use Harbor as the container registry. | **{dotted-circle}** No | +| [irker (IRC gateway)](irker.md) | Send IRC messages. | **{dotted-circle}** No | +| [Jenkins](../../../integration/jenkins.md) | Run CI/CD pipelines with Jenkins. | **{check-circle}** Yes | +| JetBrains TeamCity CI | Run CI/CD pipelines with TeamCity. | **{check-circle}** Yes | +| [Jira](../../../integration/jira/index.md) | Use Jira as the issue tracker. | **{dotted-circle}** No | +| [Mattermost notifications](mattermost.md) | Send notifications about project events to Mattermost channels. | **{dotted-circle}** No | +| [Mattermost slash commands](mattermost_slash_commands.md) | Perform common tasks with slash commands. | **{dotted-circle}** No | +| [Microsoft Teams notifications](microsoft_teams.md) | Receive event notifications. | **{dotted-circle}** No | +| Packagist | Keep your PHP dependencies updated on Packagist. | **{check-circle}** Yes | +| [Pipelines emails](pipeline_status_emails.md) | Send the pipeline status to a list of recipients by email. | **{dotted-circle}** No | +| [Pivotal Tracker](pivotal_tracker.md) | Add commit messages as comments to Pivotal Tracker stories. | **{dotted-circle}** No | +| [Prometheus](prometheus.md) | Monitor application metrics. | **{dotted-circle}** No | +| Pushover | Get real-time notifications on your device. | **{dotted-circle}** No | +| [Redmine](redmine.md) | Use Redmine as the issue tracker. | **{dotted-circle}** No | +| [Slack application](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | +| [Slack notifications](slack.md) | Send notifications about project events to Slack. | **{dotted-circle}** No | +| [Slack slash commands](slack_slash_commands.md) | Enable slash commands in a workspace. | **{dotted-circle}** No | +| [Unify Circuit](unify_circuit.md) | Send notifications about project events to Unify Circuit. | **{dotted-circle}** No | +| [Webex Teams](webex_teams.md) | Receive events notifications. | **{dotted-circle}** No | +| [YouTrack](youtrack.md) | Use YouTrack as the issue tracker. | **{dotted-circle}** No | +| [ZenTao](zentao.md) | Use ZenTao as the issue tracker. | **{dotted-circle}** No | + +### Project webhooks + +You can configure a project webhook to listen for specific events +like pushes, issues, or merge requests. When the webhook is triggered, GitLab +sends a POST request with data to a specified webhook URL. Learn more [about webhooks](webhooks.md). + +## Push hooks limit + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17874) in GitLab 12.4. + +If a single push includes changes to more than three branches or tags, integrations +supported by `push_hooks` and `tag_push_hooks` events aren't executed. + +You can change the number of supported branches or tags by changing the +[`push_event_hooks_limit` application setting](../../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). + +## Troubleshooting integrations + +Some integrations use hooks to integrate with external applications. To confirm which ones use integration hooks, see the [available integrations](#available-integrations). Learn more about [troubleshooting integration hooks](webhooks.md#troubleshoot-webhooks). + +### `Test Failed. Save Anyway` error + +Some integrations fail with an error `Test Failed. Save Anyway` when you set them +up on uninitialized repositories. This error occurs because the integration uses +push data to build the test payload, and there are no push events in the project. + +To resolve this error, initialize the repository by pushing a test file to the project +and set up the integration again. + +## Contribute to integrations + +To add a new integration, see the [Integrations development guide](../../../development/integrations/index.md). diff --git a/doc/user/project/integrations/overview.md b/doc/user/project/integrations/overview.md index 081780e6277..9625edcd8f9 100644 --- a/doc/user/project/integrations/overview.md +++ b/doc/user/project/integrations/overview.md @@ -1,117 +1,11 @@ --- -stage: Ecosystem -group: Integrations -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-07-20' --- -# Integrations **(FREE)** +This document was moved to [another location](index.md). -Integrations allow you to integrate GitLab with other applications. They -are a bit like plugins in that they allow a lot of freedom in adding -functionality to GitLab. - -## Accessing integrations - -To find the available integrations for your project: - -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Integrations**. - -There are more than 20 integrations to integrate with. Select the one that you -want to configure. - -## Integrations listing - -Click on the integration links to see further configuration instructions and details. - -| Integration | Description | Integration hooks | -| --------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ---------------------- | -| [Asana](asana.md) | Add commit messages as comments to Asana tasks. | **{dotted-circle}** No | -| Assembla | Manage projects. | **{dotted-circle}** No | -| [Atlassian Bamboo CI](bamboo.md) | Run CI/CD pipelines with Atlassian Bamboo. | **{check-circle}** Yes | -| [Bugzilla](bugzilla.md) | Use Bugzilla as the issue tracker. | **{dotted-circle}** No | -| Buildkite | Run CI/CD pipelines with Buildkite. | **{check-circle}** Yes | -| Campfire | Connect to chat. | **{dotted-circle}** No | -| [Confluence Workspace](../../../api/integrations.md#confluence-integration) | Replace the link to the internal wiki with a link to a Confluence Cloud Workspace. | **{dotted-circle}** No | -| [Custom issue tracker](custom_issue_tracker.md) | Use a custom issue tracker. | **{dotted-circle}** No | -| [Datadog](../../../integration/datadog.md) | Trace your GitLab pipelines with Datadog. | **{check-circle}** Yes | -| [Discord Notifications](discord_notifications.md) | Send notifications about project events to a Discord channel. | **{dotted-circle}** No | -| Drone CI | Run CI/CD pipelines with Drone. | **{check-circle}** Yes | -| [Emails on push](emails_on_push.md) | Send commits and diff of each push by email. | **{dotted-circle}** No | -| [EWM](ewm.md) | Use IBM Engineering Workflow Management as the issue tracker. | **{dotted-circle}** No | -| [External wiki](../wiki/index.md#link-an-external-wiki) | Link an external wiki. | **{dotted-circle}** No | -| [Flowdock](../../../api/integrations.md#flowdock) | Send notifications from GitLab to Flowdock flows. | **{dotted-circle}** No | -| [GitHub](github.md) | Obtain statuses for commits and pull requests. | **{dotted-circle}** No | -| [Google Chat](hangouts_chat.md) | Send notifications from your GitLab project to a room in Google Chat.| **{dotted-circle}** No | -| [Harbor](harbor.md) | Use Harbor as the container registry. | **{dotted-circle}** No | -| [irker (IRC gateway)](irker.md) | Send IRC messages. | **{dotted-circle}** No | -| [Jenkins](../../../integration/jenkins.md) | Run CI/CD pipelines with Jenkins. | **{check-circle}** Yes | -| JetBrains TeamCity CI | Run CI/CD pipelines with TeamCity. | **{check-circle}** Yes | -| [Jira](../../../integration/jira/index.md) | Use Jira as the issue tracker. | **{dotted-circle}** No | -| [Mattermost notifications](mattermost.md) | Send notifications about project events to Mattermost channels. | **{dotted-circle}** No | -| [Mattermost slash commands](mattermost_slash_commands.md) | Perform common tasks with slash commands. | **{dotted-circle}** No | -| [Microsoft Teams notifications](microsoft_teams.md) | Receive event notifications. | **{dotted-circle}** No | -| Packagist | Keep your PHP dependencies updated on Packagist. | **{check-circle}** Yes | -| [Pipelines emails](pipeline_status_emails.md) | Send the pipeline status to a list of recipients by email. | **{dotted-circle}** No | -| [Pivotal Tracker](pivotal_tracker.md) | Add commit messages as comments to Pivotal Tracker stories. | **{dotted-circle}** No | -| [Prometheus](prometheus.md) | Monitor application metrics. | **{dotted-circle}** No | -| Pushover | Get real-time notifications on your device. | **{dotted-circle}** No | -| [Redmine](redmine.md) | Use Redmine as the issue tracker. | **{dotted-circle}** No | -| [Slack application](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | -| [Slack notifications](slack.md) | Send notifications about project events to Slack. | **{dotted-circle}** No | -| [Slack slash commands](slack_slash_commands.md) | Enable slash commands in workspace. | **{dotted-circle}** No | -| [Unify Circuit](unify_circuit.md) | Send notifications about project events to Unify Circuit. | **{dotted-circle}** No | -| [Webex Teams](webex_teams.md) | Receive events notifications. | **{dotted-circle}** No | -| [YouTrack](youtrack.md) | Use YouTrack as the issue tracker. | **{dotted-circle}** No | -| [ZenTao](zentao.md) | Use ZenTao as the issue tracker. | **{dotted-circle}** No | - -## Push hooks limit - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17874) in GitLab 12.4. - -If a single push includes changes to more than three branches or tags, integrations -supported by `push_hooks` and `tag_push_hooks` events aren't executed. - -The number of branches or tags supported can be changed via -[`push_event_hooks_limit` application setting](../../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). - -## Project integration management - -Project integration management lets you control integration settings across all projects -of an instance. On the project level, administrators you can choose whether to inherit the -instance configuration or provide custom settings. - -Read more about [Project integration management](../../admin_area/settings/project_integration_management.md). - -## SSL verification - -By default, the SSL certificate for outgoing HTTP requests is verified based on -an internal list of Certificate Authorities. This means the certificate cannot -be self-signed. - -You can turn off SSL verification in the configuration settings for [webhooks](webhooks.md#configure-a-webhook-in-gitlab) -and some integrations. - -## Troubleshooting integrations - -Some integrations use hooks for integration with external applications. To confirm which ones use integration hooks, see the [integrations listing](#integrations-listing) above. Learn more about [troubleshooting integration hooks](webhooks.md#troubleshoot-webhooks). - -### Uninitialized repositories - -Some integrations fail with an error `Test Failed. Save Anyway` when you attempt to set them up on -uninitialized repositories. Some integrations use push data to build the test payload, -and this error occurs when no push events exist in the project yet. - -To resolve this error, initialize the repository by pushing a test file to the project and set up -the integration again. - -## Contributing to integrations - -Because GitLab is open source we can ship with the code and tests for all -plugins. This allows the community to keep the plugins up to date so that they -always work in newer GitLab versions. - -For an overview of what integrations are available, please see the -[integrations source directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/app/models/integrations). - -Contributions are welcome! +<!-- This redirect file can be deleted after 2022-07-20. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index f4f5b3f545b..ac7d447961c 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -57,7 +57,7 @@ You can configure a webhook for a group or a project. The URL must be percent-encoded if it contains one or more special characters. 1. In **Secret token**, enter the [secret token](#validate-payloads-by-using-a-secret-token) to validate payloads. 1. In the **Trigger** section, select the [events](webhook_events.md) to trigger the webhook. -1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](overview.md#ssl-verification). +1. Optional. Clear the **Enable SSL verification** checkbox to disable [SSL verification](index.md#manage-ssl-verification). 1. Select **Add webhook**. ## Configure your webhook receiver endpoint @@ -244,7 +244,7 @@ To view the table: - **Failed to connect** if it is misconfigured, and needs manual intervention to re-enable it. - **Fails to connect** if it is temporarily disabled and will retry later. - +  1. Select **Edit** for the webhook you want to view. diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 15130523da6..9d23a63b940 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -36,7 +36,7 @@ The second way is to locate the Confidentiality section in the sidebar and click | Turn off confidentiality | Turn on confidentiality | | :-----------: | :----------: | -|  |  | +|  |  | Every change from regular to confidential and vice versa, is indicated by a system note in the issue's comments. @@ -97,5 +97,5 @@ sees in the project's search results respectively. - [Merge requests for confidential issues](../merge_requests/confidential.md) - [Make an epic confidential](../../group/epics/manage_epics.md#make-an-epic-confidential) -- [Mark a comment as confidential](../../discussions/index.md#mark-a-comment-as-confidential) +- [Add an internal note](../../discussions/index.md#add-an-internal-note) - [Security practices for confidential merge requests](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) at GitLab diff --git a/doc/user/project/issues/csv_import.md b/doc/user/project/issues/csv_import.md index e4b8efd27ed..a3f6ee5e61e 100644 --- a/doc/user/project/issues/csv_import.md +++ b/doc/user/project/issues/csv_import.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Importing issues from CSV **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23532) in GitLab 11.7. - Issues can be imported to a project by uploading a CSV file with the columns `title` and `description`. Other columns are **not** imported. If you want to retain columns such as labels and milestones, consider the [Move Issue feature](managing_issues.md#move-an-issue). @@ -25,33 +23,29 @@ You must have at least the Developer role for a project to import issues. To import issues: -1. Navigate to a project's Issues list page. -1. If existing issues are present, click the import icon at the top right, next to the **Edit issues** button. -1. For a project without any issues, click the button labeled **Import CSV** in the middle of the page. -1. Select the file and click the **Import issues** button. +1. Go to your project's Issues list page. +1. Open the import feature, depending if the project has issues: + - Existing issues are present: Select the import icon at the top right, next to **Edit issues**. + - Project has no issues: Select **Import CSV** in the middle of the page. +1. Select the file you want to import, and then select **Import issues**. -The file is processed in the background and a notification email is sent -to you once the import is complete. +The file is processed in the background, and a notification email is sent +to you after the import is complete. ## CSV file format -When importing issues from a CSV file, it must be formatted in a certain way: +To import issues, GitLab requires CSV files have a specific format: -- **header row:** CSV files must include the following headers: -`title` and `description`. The case of the headers does not matter. -- **columns:** Data from columns beyond `title` and `description` are not imported. -- **separators:** The column separator is automatically detected from the header row. - Supported separator characters are: commas (`,`), semicolons (`;`), and tabs (`\t`). - The row separator can be either `CRLF` or `LF`. -- **double-quote character:** The double-quote (`"`) character is used to quote fields, - enabling the use of the column separator within a field (see the third line in the - sample CSV data below). To insert a double-quote (`"`) within a quoted - field, use two double-quote characters in succession (`""`). -- **data rows:** After the header row, succeeding rows must follow the same column - order. The issue title is required while the description is optional. +| Element | Format | +|------------------------|--------| +| header row | CSV files must include the following headers: `title` and `description`. The case of the headers does not matter. | +| columns | Data from columns beyond `title` and `description` are not imported. | +| separators | The column separator is detected from the header row. Supported separator characters are commas (`,`), semicolons (`;`), and tabs (`\t`). The row separator can be either `CRLF` or `LF`. | +| double-quote character | The double-quote (`"`) character is used to quote fields, enabling the use of the column separator in a field (see the third line in the sample CSV data below). To insert a double-quote (`"`) in a quoted field use two double-quote characters in succession (`""`). | +| data rows | After the header row, following rows must use the same column order. The issue title is required, but the description is optional. | -If you have special characters _within_ a field, (such as `\n` or `,`), -wrap the characters in double quotes. +If you have special characters in a field, (such as `\n` or `,`), surround the +characters with double quotes (`"`). Sample CSV data: @@ -64,6 +58,7 @@ Another Title,"A description, with a comma" ### File size -The limit depends on the configuration value of Max Attachment Size for the GitLab instance. +The limit depends on how your GitLab instance is hosted: -For GitLab.com, it is set to 10 MB. +- Self-managed: Set by the configuration value of `Max Attachment Size` for the GitLab instance. +- GitLab SaaS: On GitLab.com, it's set to 10 MB. diff --git a/doc/user/project/issues/img/confidential_issues_issue_page.png b/doc/user/project/issues/img/confidential_issues_issue_page.png Binary files differindex 0f5c774d258..b349149aa98 100644 --- a/doc/user/project/issues/img/confidential_issues_issue_page.png +++ b/doc/user/project/issues/img/confidential_issues_issue_page.png diff --git a/doc/user/project/issues/img/design_management_v14_10.png b/doc/user/project/issues/img/design_management_v14_10.png Binary files differindex a10be15eafd..133c0a52d6b 100644 --- a/doc/user/project/issues/img/design_management_v14_10.png +++ b/doc/user/project/issues/img/design_management_v14_10.png diff --git a/doc/user/project/issues/img/sidebar_confidential_issue.png b/doc/user/project/issues/img/sidebar_confidential_issue.png Binary files differindex a320f4dcfe5..0ef61c7f1b0 100644 --- a/doc/user/project/issues/img/sidebar_confidential_issue.png +++ b/doc/user/project/issues/img/sidebar_confidential_issue.png diff --git a/doc/user/project/issues/img/turn_off_confidentiality.png b/doc/user/project/issues/img/turn_off_confidentiality.png Binary files differdeleted file mode 100644 index 04a85933071..00000000000 --- a/doc/user/project/issues/img/turn_off_confidentiality.png +++ /dev/null diff --git a/doc/user/project/issues/img/turn_off_confidentiality_v15_0.png b/doc/user/project/issues/img/turn_off_confidentiality_v15_0.png Binary files differnew file mode 100644 index 00000000000..37cbe0f4fd9 --- /dev/null +++ b/doc/user/project/issues/img/turn_off_confidentiality_v15_0.png diff --git a/doc/user/project/issues/img/turn_on_confidentiality.png b/doc/user/project/issues/img/turn_on_confidentiality.png Binary files differdeleted file mode 100644 index fac360ca6dc..00000000000 --- a/doc/user/project/issues/img/turn_on_confidentiality.png +++ /dev/null diff --git a/doc/user/project/issues/img/turn_on_confidentiality_v15_0.png b/doc/user/project/issues/img/turn_on_confidentiality_v15_0.png Binary files differnew file mode 100644 index 00000000000..498867d1933 --- /dev/null +++ b/doc/user/project/issues/img/turn_on_confidentiality_v15_0.png diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 4508ef30ac5..7db66dd013b 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -225,6 +225,8 @@ When you're creating a new issue, you can complete the following fields: ## Edit an issue +> Reordering list items [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.0. + You can edit an issue's title and description. Prerequisites: @@ -237,6 +239,14 @@ To edit an issue: 1. Edit the available fields. 1. Select **Save changes**. +You can also reorder list items, which include bullet, numerical, and task list items. +To reorder list items: + +1. Hover over the list item row to make the drag icon visible. +1. Click and hold the drag icon. +1. Drag the row to the new position in the list. +1. Release the drag icon. + ### Bulk edit issues from a project > - Assigning epic [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210470) in GitLab 13.2. diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index ff4677eddde..28bd353d8cc 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -189,7 +189,7 @@ You can filter and sort members in a project. 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Project information > Members**. 1. In the **Filter members** box, select `Membership` `=` `Inherited`. -1. Press Enter. +1. Press <kbd>Enter</kbd>.  @@ -198,7 +198,7 @@ You can filter and sort members in a project. 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Project information > Members**. 1. In the **Filter members** box, select `Membership` `=` `Direct`. -1. Press Enter. +1. Press <kbd>Enter</kbd>.  diff --git a/doc/user/project/merge_requests/allow_collaboration.md b/doc/user/project/merge_requests/allow_collaboration.md index 5826ebcab49..d06c8182e22 100644 --- a/doc/user/project/merge_requests/allow_collaboration.md +++ b/doc/user/project/merge_requests/allow_collaboration.md @@ -60,10 +60,10 @@ In the following example: To change or add a commit to the contributor's merge request: -1. Open the merge request page, and select the **Overview** tab. -1. Scroll to the merge request widget, and select **Check out branch**. +1. Go to the merge request. +1. In the upper right corner, select **Code**, then select **Check out branch**. 1. In the modal window, select **Copy** (**{copy-to-clipboard}**). -1. In your terminal, navigate to your cloned version of the repository, and +1. In your terminal, go to your cloned version of the repository, and paste the commands. For example: ```shell diff --git a/doc/user/project/merge_requests/approvals/img/security_approvals_v15_0.png b/doc/user/project/merge_requests/approvals/img/security_approvals_v15_0.png Binary files differnew file mode 100644 index 00000000000..b28d216f180 --- /dev/null +++ b/doc/user/project/merge_requests/approvals/img/security_approvals_v15_0.png diff --git a/doc/user/project/merge_requests/approvals/index.md b/doc/user/project/merge_requests/approvals/index.md index e940426dc67..f0ab4d606ad 100644 --- a/doc/user/project/merge_requests/approvals/index.md +++ b/doc/user/project/merge_requests/approvals/index.md @@ -102,8 +102,8 @@ Without the approvals, the work cannot merge. Required approvals enable multiple - Use the [code owners of changed files](rules.md#code-owners-as-eligible-approvers), to determine who should review the work. - Require an [approval before merging code that causes test coverage to decline](../../../../ci/pipelines/settings.md#coverage-check-approval-rule) -- [Require approval from a security team](../../../application_security/index.md#security-approvals-in-merge-requests) - before merging code that could introduce a vulnerability. +- Users on GitLab Ultimate can also [require approval from a security team](../../../application_security/index.md#security-approvals-in-merge-requests) + before merging code that could introduce a vulnerability. ## Related topics diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index 01772e59127..17a42e1b540 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -242,3 +242,14 @@ the API. For more information about this validation error, read [issue 285129](https://gitlab.com/gitlab-org/gitlab/-/issues/285129). + +## Security Approvals **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357021) in GitLab 15.0. + +You can use [scan result policies](../../../application_security/policies/scan-result-policies.md#scan-result-policy-editor) to define security approvals based on the status of vulnerabilities in the merge request and the default branch. +Details for each security policy is shown in the Security Approvals section of your Merge Request configuration. + + + +These policies are both created and edited in the [security policy editor](../../../application_security/policies/index.md#policy-editor). diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index 0ede9310393..9c2b54888fb 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -119,16 +119,9 @@ when more changes are added to it: 1. Select the **Remove all approvals when commits are added to the source branch** checkbox. 1. Select **Save changes**. -Approvals aren't reset when a merge request is [rebased from the UI](../fast_forward_merge.md) +Approvals aren't reset when a merge request is [rebased from the UI](../methods/index.md#rebasing-in-semi-linear-merge-methods) However, approvals are reset if the target branch is changed. -## Security approvals in merge requests **(ULTIMATE)** - -You can require that a member of your security team approves a merge request if a -merge request could introduce a vulnerability. - -To learn more, see [Security approvals in merge requests](../../../application_security/index.md#security-approvals-in-merge-requests). - ## Code coverage check approvals You can require specific approvals if a merge request would result in a decline in code test diff --git a/doc/user/project/merge_requests/changes.md b/doc/user/project/merge_requests/changes.md index 8796ea0635b..5016a33ed28 100644 --- a/doc/user/project/merge_requests/changes.md +++ b/doc/user/project/merge_requests/changes.md @@ -14,7 +14,7 @@ changes. By default, the diff view compares the versions of files in the merge request source branch to the files in the target branch, and shows only the parts of a file that have changed. - + ## Show all changes in a merge request diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index 10fc778d5ae..7e8ef9272d4 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -72,7 +72,7 @@ This example shows how to run Code Quality on your code by using GitLab CI/CD an In either configuration, the runner must have enough disk space to handle generated Code Quality files. For example on the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. -Once you set up GitLab Runner, include the Code Quality template in your CI configuration: +Once you set up GitLab Runner, include the [Code Quality template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml) in your CI configuration: ```yaml include: @@ -257,9 +257,9 @@ The template has these [`rules`](../../../ci/yaml/index.md#rules) for the `code ```yaml code_quality: rules: - - if: '$CODE_QUALITY_DISABLED' + - if: $CODE_QUALITY_DISABLED when: never - - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + - if: $CI_COMMIT_TAG || $CI_COMMIT_BRANCH ``` If you are using merge request pipelines, your `rules` (or [`workflow: rules`](../../../ci/yaml/index.md#workflow)) @@ -268,9 +268,9 @@ might look like this example: ```yaml job1: rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Run job1 in merge request pipelines - - if: '$CI_COMMIT_BRANCH == "main"' # Run job1 in pipelines on the main branch (but not in other branch pipelines) - - if: '$CI_COMMIT_TAG' # Run job1 in pipelines for tags + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Run job1 in merge request pipelines + - if: $CI_COMMIT_BRANCH == "main" # Run job1 in pipelines on the main branch (but not in other branch pipelines) + - if: $CI_COMMIT_TAG # Run job1 in pipelines for tags ``` To make these work together, you need to overwrite the code quality `rules` @@ -282,11 +282,11 @@ include: code_quality: rules: - - if: '$CODE_QUALITY_DISABLED' + - if: $CODE_QUALITY_DISABLED when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Run code quality job in merge request pipelines - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # Run code quality job in pipelines on the default branch (but not in other branch pipelines) - - if: '$CI_COMMIT_TAG' # Run code quality job in pipelines for tags + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Run code quality job in merge request pipelines + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run code quality job in pipelines on the default branch (but not in other branch pipelines) + - if: $CI_COMMIT_TAG # Run code quality job in pipelines for tags ``` ### Configure Code Quality to use a private container image registry @@ -541,7 +541,7 @@ This can be due to multiple reasons: - If no [degradation or error is detected](https://docs.codeclimate.com/docs/maintainability#section-checks), nothing is displayed. - The [`artifacts:expire_in`](../../../ci/yaml/index.md#artifactsexpire_in) CI/CD - setting can cause the Code Quality artifact(s) to expire faster than desired. + setting can cause the Code Quality artifacts to expire faster than desired. - The widgets use the pipeline of the latest commit to the target branch. If commits are made to the default branch that do not run the code quality job, this may cause the merge request widget to have no base report for comparison. - If you use the [`REPORT_STDOUT` environment variable](https://gitlab.com/gitlab-org/ci-cd/codequality#environment-variables), no report file is generated and nothing displays in the merge request. - Large `gl-code-quality-report.json` files (esp. >10 MB) are [known to prevent the report from being displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/2737). diff --git a/doc/user/project/merge_requests/confidential.md b/doc/user/project/merge_requests/confidential.md index 6900880417f..5b17ec009e4 100644 --- a/doc/user/project/merge_requests/confidential.md +++ b/doc/user/project/merge_requests/confidential.md @@ -74,5 +74,5 @@ Open a merge request - [Confidential issues](../issues/confidential_issues.md) - [Make an epic confidential](../../group/epics/manage_epics.md#make-an-epic-confidential) -- [Mark a comment as confidential](../../discussions/index.md#mark-a-comment-as-confidential) +- [Add an internal note](../../discussions/index.md#add-an-internal-note) - [Security practices for confidential merge requests](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) at GitLab diff --git a/doc/user/project/merge_requests/fast_forward_merge.md b/doc/user/project/merge_requests/fast_forward_merge.md index 77162aa0b83..048421a3a5b 100644 --- a/doc/user/project/merge_requests/fast_forward_merge.md +++ b/doc/user/project/merge_requests/fast_forward_merge.md @@ -1,74 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference, concepts +redirect_to: 'methods/index.md' +remove_date: '2022-08-09' --- -# Fast-forward merge requests **(FREE)** +This document was moved to [another location](methods/index.md). -Sometimes, a workflow policy might mandate a clean commit history without -merge commits. In such cases, the fast-forward merge is the perfect candidate. - -With fast-forward merge requests, you can retain a linear Git history and a way -to accept merge requests without creating merge commits. - -When the fast-forward merge -([`--ff-only`](https://git-scm.com/docs/git-merge#git-merge---ff-only)) setting -is enabled, no merge commits are created and all merges are fast-forwarded, -which means that merging is only allowed if the branch can be fast-forwarded. - -When a fast-forward merge is not possible, the user is given the option to rebase. - -NOTE: -Projects using the fast-forward merge strategy can't filter merge requests -[by deployment date](../../search/index.md#filtering-merge-requests-by-environment-or-deployment-date), -because no merge commit is created. - -## Enabling fast-forward merges - -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > General**. -1. Expand **Merge requests**. -1. In the **Merge method** section, select **Fast-forward merge**. -1. Select **Save changes**. - -Now, when you visit the merge request page, you can accept it -**only if a fast-forward merge is possible**. - - - -If a fast-forward merge is not possible but a conflict free rebase is possible, -a rebase button is offered. - -You can also rebase without running a CI/CD pipeline. -[Introduced in](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) GitLab 14.7. - -The rebase action is also available as a [quick action command: `/rebase`](../../../topics/git/git_rebase.md#rebase-from-the-gitlab-ui). - - - -If the target branch is ahead of the source branch and a conflict free rebase is -not possible, you need to rebase the -source branch locally before you can do a fast-forward merge. - - - -## Fast-forward merges prevent squashing commits - -If your project has enabled fast-forward merges, to merge cleanly, the code in a -merge request cannot use [squashing during merge](squash_and_merge.md). Squashing -is available only when accepting a merge request. Rebasing may be required before -squashing, even though squashing can itself be considered equivalent to rebasing. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +<!-- This redirect file can be deleted after <2022-08-09>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index fd1751585d5..c1986a80ca0 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -51,9 +51,9 @@ Learn the various ways to [create a merge request](creating_merge_requests.md). When you start a new merge request, you can immediately include the following options. You can also add them later by either selecting **Edit** on the merge request's page at the top-right side, or by using -[keyboard shortcuts for merge requests](../../shortcuts.md#issues-and-merge-requests): +[keyboard shortcuts for merge requests](../../shortcuts.md#merge-requests): -- [Assign](#assignee) the merge request to a colleague for review. With [multiple assignees](#multiple-assignees), you can assign it to more than one person at a time. +- [Assign](index.md#assign-a-user-to-a-merge-request) the merge request to a colleague for review. With [multiple assignees](index.md#assign-multiple-users), you can assign it to more than one person at a time. - Set a [milestone](../milestones/index.md) to track time-sensitive changes. - Add [labels](../labels.md) to help contextualize and filter your merge requests over time. - [Require approval](approvals/index.md#required-approvals) from your team. @@ -76,43 +76,11 @@ After you have created the merge request, you can also: Many of these options can be set: -- From the merge request page, with [keyboard shortcuts](../../shortcuts.md#issues-and-merge-requests). +- From the merge request page, with [keyboard shortcuts](../../shortcuts.md#merge-requests). - When pushing changes from the command line, with [Git push options](../push_options.md). See also other [features associated to merge requests](reviews/index.md#associated-features). -### Assignee - -Choose an assignee to designate someone as the person responsible -for the first [review of the merge request](reviews/index.md). -Open the drop down box to search for the user you wish to assign, -and the merge request is added to their -[assigned merge request list](../../search/index.md#search-issues-and-merge-requests). - -#### Multiple assignees **(PREMIUM)** - -> Moved to GitLab Premium in 13.9 - -Multiple people often review merge requests at the same time. -GitLab allows you to have multiple assignees for merge requests -to indicate everyone that is reviewing or accountable for it. - - - -To assign multiple assignees to a merge request: - -1. From a merge request, expand the right sidebar and locate the **Assignees** section. -1. Click on **Edit** and from the dropdown menu, select as many users as you want - to assign the merge request to. - -Similarly, assignees are removed by deselecting them from the same -dropdown menu. - -It is also possible to manage multiple assignees: - -- When creating a merge request. -- Using [quick actions](../quick_actions.md#issues-merge-requests-and-epics). - ### Reviewer WARNING: diff --git a/doc/user/project/merge_requests/img/merge_method_ff_v15_0.png b/doc/user/project/merge_requests/img/merge_method_ff_v15_0.png Binary files differnew file mode 100644 index 00000000000..323fd03ffa2 --- /dev/null +++ b/doc/user/project/merge_requests/img/merge_method_ff_v15_0.png diff --git a/doc/user/project/merge_requests/img/merge_method_merge_commit_v15_0.png b/doc/user/project/merge_requests/img/merge_method_merge_commit_v15_0.png Binary files differnew file mode 100644 index 00000000000..b880c2c0e04 --- /dev/null +++ b/doc/user/project/merge_requests/img/merge_method_merge_commit_v15_0.png diff --git a/doc/user/project/merge_requests/img/merge_method_merge_commit_with_semi_linear_history_v15_0.png b/doc/user/project/merge_requests/img/merge_method_merge_commit_with_semi_linear_history_v15_0.png Binary files differnew file mode 100644 index 00000000000..9eab71e9d3c --- /dev/null +++ b/doc/user/project/merge_requests/img/merge_method_merge_commit_with_semi_linear_history_v15_0.png diff --git a/doc/user/project/merge_requests/img/mr-diff-example_v14_8.png b/doc/user/project/merge_requests/img/mr-diff-example_v14_8.png Binary files differdeleted file mode 100644 index 1984defde9a..00000000000 --- a/doc/user/project/merge_requests/img/mr-diff-example_v14_8.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/mr-diff-example_v15.png b/doc/user/project/merge_requests/img/mr-diff-example_v15.png Binary files differnew file mode 100644 index 00000000000..8fdf3935906 --- /dev/null +++ b/doc/user/project/merge_requests/img/mr-diff-example_v15.png diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index a3b9fb52f0d..510dcd82907 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -70,6 +70,72 @@ change and whether you need access to a development environment: - [Push changes from the command line](../../../gitlab-basics/start-using-git.md), if you are familiar with Git and the command line. +## Assign a user to a merge request + +When a merge request is created, it's assigned by default to the person who created it. +This person owns the merge request, but isn't responsible for [reviewing it](reviews/index.md). +To assign the merge request to someone else, use the `/assign @user` +[quick action](../quick_actions.md#issues-merge-requests-and-epics) in a text area in +a merge request, or: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** and find your merge request. +1. On the right sidebar, expand the right sidebar and locate the **Assignees** section. +1. Select **Edit**. +1. Search for the user you want to assign, and select the user. + +The merge request is added to the user's +[assigned merge request list](../../search/index.md#search-issues-and-merge-requests). + +### Assign multiple users **(PREMIUM)** + +> Moved to GitLab Premium in 13.9. + +GitLab enables multiple assignees for merge requests, if multiple people are +accountable for it: + + + +To assign multiple assignees to a merge request, use the `/assign @user` +[quick action](../quick_actions.md#issues-merge-requests-and-epics) in a text area, or: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** and find your merge request. +1. On the right sidebar, expand the right sidebar and locate the **Assignees** section. +1. Select **Edit** and, from the dropdown list, select all users you want + to assign the merge request to. + +To remove an assignee, clear the user from the same dropdown list. + +## Close a merge request + +If you decide to permanently stop work on a merge request, +GitLab recommends you close the merge request rather than +[delete it](#delete-a-merge-request). The author and assignees of a merge request, and users with +Developer, Maintainer, or Owner [roles](../../permissions.md) in a project +can close merge requests in the project: + +1. Go to the merge request you want to close. +1. Scroll to the comment box at the bottom of the page. +1. Following the comment box, select **Close merge request**. + +GitLab closes the merge request, but preserves records of the merge request, +its comments, and any associated pipelines. + +### Delete a merge request + +GitLab recommends you close, rather than delete, merge requests. + +WARNING: +You cannot undo the deletion of a merge request. + +To delete a merge request: + +1. Sign in to GitLab as a user with the project Owner role. + Only users with this role can delete merge requests in a project. +1. Go to the merge request you want to delete, and select **Edit**. +1. Scroll to the bottom of the page, and select **Delete merge request**. + ## Request attention to a merge request > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343528) in GitLab 14.10 [with a flag](../../../administration/feature_flags.md) named `mr_attention_requests`. Disabled by default. @@ -117,35 +183,6 @@ only one attention request, which is synced across both duties. If the attention request is removed from you, either as an assignee or a reviewer, it is removed from both your duties. -## Close a merge request - -If you decide to permanently stop work on a merge request, -GitLab recommends you close the merge request rather than -[delete it](#delete-a-merge-request). The author and assignees of a merge request, and users with -Developer, Maintainer, or Owner [roles](../../permissions.md) in a project -can close merge requests in the project: - -1. Go to the merge request you want to close. -1. Scroll to the comment box at the bottom of the page. -1. Following the comment box, select **Close merge request**. - -GitLab closes the merge request, but preserves records of the merge request, -its comments, and any associated pipelines. - -### Delete a merge request - -GitLab recommends you close, rather than delete, merge requests. - -WARNING: -You cannot undo the deletion of a merge request. - -To delete a merge request: - -1. Sign in to GitLab as a user with the project Owner role. - Only users with this role can delete merge requests in a project. -1. Go to the merge request you want to delete, and select **Edit**. -1. Scroll to the bottom of the page, and select **Delete merge request**. - ## Merge request workflows For a software developer working in a team: diff --git a/doc/user/project/merge_requests/load_performance_testing.md b/doc/user/project/merge_requests/load_performance_testing.md index 7861e1e28fc..a5fff4a38be 100644 --- a/doc/user/project/merge_requests/load_performance_testing.md +++ b/doc/user/project/merge_requests/load_performance_testing.md @@ -189,7 +189,7 @@ review: paths: - review.env rules: - - if: '$CI_COMMIT_BRANCH' # Modify to match your pipeline rules, or use `only/except` if needed. + - if: $CI_COMMIT_BRANCH # Modify to match your pipeline rules, or use `only/except` if needed. load_performance: dependencies: @@ -197,5 +197,5 @@ load_performance: variables: K6_DOCKER_OPTIONS: '--env-file review.env' rules: - - if: '$CI_COMMIT_BRANCH' # Modify to match your pipeline rules, or use `only/except` if needed. + - if: $CI_COMMIT_BRANCH # Modify to match your pipeline rules, or use `only/except` if needed. ``` diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index 256dde4fa17..ac1c61f2e72 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -50,7 +50,7 @@ You can prevent merge requests from being merged if: This works for both: - GitLab CI/CD pipelines -- Pipelines run from an [external CI integration](../integrations/overview.md#integrations-listing) +- Pipelines run from an [external CI integration](../integrations/index.md#available-integrations) As a result, [disabling GitLab CI/CD pipelines](../../../ci/enable_or_disable_ci.md) does not disable this feature, as it is possible to use pipelines from external @@ -81,13 +81,13 @@ it could allow code that fails tests to be merged: ```yaml branch-pipeline-job: rules: - - if: '$CI_PIPELINE_SOURCE == "push"' + - if: $CI_PIPELINE_SOURCE == "push" script: - echo "Code testing scripts here, for example." merge-request-pipeline-job: rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + - if: $CI_PIPELINE_SOURCE == "merge_request_event" script: - echo "No tests run, but this pipeline always succeeds and enables merge." - echo true diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md new file mode 100644 index 00000000000..adfa5288f81 --- /dev/null +++ b/doc/user/project/merge_requests/methods/index.md @@ -0,0 +1,116 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference, concepts +--- + +# Merge methods **(FREE)** + +The merge method you select for your project determines how the changes in your +merge requests are merged into an existing branch. + +## Configure a project's merge method + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Merge requests**. +1. In the **Merge method** section, select your desired merge method. +1. Select **Save changes**. + +## Merge commit + +This setting is the default. It always creates a separate merge commit, +even when using [squash](../squash_and_merge.md). An example commit graph generated using this merge method: + + + +- For regular merges, it is equivalent to the command `git merge --no-ff <source-branch>`. +- For squash merges, it squashes all commits in the source branch before merging it normally. It performs actions similar to: + + ```shell + git checkout `git merge-base <source-branch> <target-branch>` + git merge --squash <source-branch> + SOURCE_SHA=`git rev-parse HEAD` + git checkout <target-branch> + git merge --no-ff $SOURCE_SHA + ``` + +## Merge commit with semi-linear history + +A merge commit is created for every merge, but the branch is only merged if +a fast-forward merge is possible. This ensures that if the merge request build +succeeded, the target branch build also succeeds after the merge. An example commit graph generated using this merge method: + + + +When you visit the merge request page with `Merge commit with semi-linear history` +method selected, you can accept it **only if a fast-forward merge is possible**. +When a fast-forward merge is not possible, the user is given the option to rebase, see +[Rebasing in (semi-)linear merge methods](#rebasing-in-semi-linear-merge-methods). + +This method is equivalent to the same Git commands as in the **Merge commit** method. However, +if your source branch is based on an out-of-date version of the target branch (such as `main`), +you must rebase your source branch. +This merge method creates a cleaner-looking history, while still enabling you to +see where every branch began and was merged. + +## Fast-forward merge + +Sometimes, a workflow policy might mandate a clean commit history without +merge commits. In such cases, the fast-forward merge is appropriate. With +fast-forward merge requests, you can retain a linear Git history and a way +to accept merge requests without creating merge commits. An example commit graph +generated using this merge method: + + + +This method is equivalent to `git merge --ff <source-branch>` for regular merges, and to +`git merge -squash <source-branch>` for squash merges. + +When the fast-forward merge +([`--ff-only`](https://git-scm.com/docs/git-merge#git-merge---ff-only)) setting +is enabled, no merge commits are created and all merges are fast-forwarded, +which means that merging is only allowed if the branch can be fast-forwarded. +When a fast-forward merge is not possible, the user is given the option to rebase, see +[Rebasing in (semi-)linear merge methods](#rebasing-in-semi-linear-merge-methods). + +NOTE: +Projects using the fast-forward merge strategy can't filter merge requests +[by deployment date](../../../search/index.md#filtering-merge-requests-by-environment-or-deployment-date), +because no merge commit is created. + +When you visit the merge request page with `Fast-forward merge` +method selected, you can accept it **only if a fast-forward merge is possible**. + + + +## Rebasing in (semi-)linear merge methods + +> Rebasing without running a CI/CD pipeline [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) in GitLab 14.7. + +In these merge methods, you can merge only when your source branch is up-to-date with the target branch: + +- Merge commit with semi-linear history. +- Fast-forward merge. + +If a fast-forward merge is not possible but a conflict-free rebase is possible, +GitLab offers you the [`/rebase` quick action](../../../../topics/git/git_rebase.md#rebase-from-the-gitlab-ui), +and the ability to **Rebase** from the user interface: + + + +In [GitLab 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) and later, you can also rebase without running a CI/CD pipeline. + +If the target branch is ahead of the source branch and a conflict-free rebase is +not possible, you must rebase the source branch locally before you can do a fast-forward merge. + + + +Rebasing may be required before squashing, even though squashing can itself be +considered equivalent to rebasing. + +## Related topics + +- [Commits history](../commits.md) +- [Squash and merge](../squash_and_merge.md) diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index 6441ccb73fe..7b4a41f9339 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -14,7 +14,7 @@ by clicking the **Revert** button in merge requests and commit details. NOTE: The **Revert** button is shown only for projects that use the merge method "Merge Commit", which can be set under the project's -**Settings > General > Merge request**. [Fast-forward commits](fast_forward_merge.md) +**Settings > General > Merge request**. [Fast-forward commits](methods/index.md#fast-forward-merge) can not be reverted by using the merge request view. After the merge request has been merged, use the **Revert** button diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index 512faae82a9..eb5a54e6119 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -131,17 +131,6 @@ the author of the merge request can request a new review from the reviewer: GitLab creates a new [to-do item](../../../todos.md) for the reviewer, and sends them a notification email. -## Semi-linear history merge requests - -A merge commit is created for every merge, but the branch is only merged if -a fast-forward merge is possible. This ensures that if the merge request build -succeeded, the target branch build also succeeds after the merge. - -1. Go to your project and select **Settings > General**. -1. Expand **Merge requests**. -1. In the **Merge method** section, select **Merge commit with semi-linear history**. -1. Select **Save changes**. - ## Comment on multiple lines > - [Introduced](https://gitlab.com/gitlab-org/ux-research/-/issues/870) in GitLab 13.2. @@ -211,17 +200,17 @@ These features are associated with merge requests: - [Cherry-pick changes](../cherry_pick_changes.md): Cherry-pick any commit in the UI by selecting the **Cherry-pick** button in a merged merge requests or a commit. -- [Fast-forward merge requests](../fast_forward_merge.md): +- [Fast-forward merge requests](../methods/index.md#fast-forward-merge): For a linear Git history and a way to accept merge requests without creating merge commits - [Find the merge request that introduced a change](../versions.md): - When viewing the commit details page, GitLab links to the merge request(s) containing that commit. + When viewing the commit details page, GitLab links to the merge requests containing that commit. - [Merge requests versions](../versions.md): Select and compare the different versions of merge request diffs - [Resolve conflicts](../conflicts.md): GitLab can provide the option to resolve certain merge request conflicts in the GitLab UI. - [Revert changes](../revert_changes.md): Revert changes from any commit from a merge request. -- [Keyboard shortcuts](../../../shortcuts.md#issues-and-merge-requests): +- [Keyboard shortcuts](../../../shortcuts.md#merge-requests): Access and modify specific parts of a merge request with keyboard commands. ## Troubleshooting @@ -365,3 +354,7 @@ All the above can be done with the [`git-mr`](https://gitlab.com/glensc/git-mr) In a group, the sidebar displays the total count of open merge requests. This value is cached if it's greater than than 1000. The cached value is rounded to thousands (or millions) and updated every 24 hours. + +## Related topics + +- [Merge methods](../methods/index.md) diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index a1d6959b75e..7e37990b9bf 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -18,8 +18,8 @@ in your Git repository by using the _squash and merge_ strategy. Each time a branch merges into your base branch, up to two commits are added: - The single commit created by squashing the commits from the branch. -- A merge commit, unless you have [enabled fast-forward merges](fast_forward_merge.md#enabling-fast-forward-merges) - in your project. Fast-forward merges disable both merge commits and squashing. +- A merge commit, unless you have enabled [fast-forward merges](methods/index.md#fast-forward-merge) + in your project. Fast-forward merges disable merge commits. By default, squashed commits contain the following metadata: @@ -74,7 +74,7 @@ To configure the default squashing behavior for all merge requests in your proje ## Related topics - [Commit message templates](commit_templates.md) -- [Fast-forward merges](fast_forward_merge.md) +- [Merge methods](methods/index.md) <!-- ## Troubleshooting diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 9f1e5ae7046..85b5bbea284 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -28,7 +28,7 @@ between pipeline completion and the visualization loading on the page. For the coverage analysis to work, you have to provide a properly formatted [Cobertura XML](https://cobertura.github.io/cobertura/) report to -[`artifacts:reports:cobertura`](../../../ci/yaml/artifacts_reports.md#artifactsreportscobertura-deprecated). +[`artifacts:reports:coverage_report`](../../../ci/yaml/artifacts_reports.md#artifactsreportscoverage_report). This format was originally developed for Java, but most coverage analysis frameworks for other languages have plugins to add support for it, like: @@ -196,7 +196,9 @@ coverage-jdk11: needs: ["test-jdk11"] artifacts: reports: - cobertura: target/site/cobertura.xml + coverage_report: + coverage_format: cobertura + path: target/site/cobertura.xml ``` #### Gradle example @@ -232,7 +234,9 @@ coverage-jdk11: needs: ["test-jdk11"] artifacts: reports: - cobertura: build/cobertura.xml + coverage_report: + coverage_format: cobertura + path: build/cobertura.xml ``` ### Python example @@ -251,9 +255,12 @@ run tests: - coverage run -m pytest - coverage report - coverage xml + coverage: '/TOTAL.*\s([.\d]+)%/' artifacts: reports: - cobertura: coverage.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml ``` ### PHP example @@ -263,7 +270,7 @@ to collect test coverage data and generate the report. With a minimal [`phpunit.xml`](https://phpunit.readthedocs.io/en/9.5/configuration.html) file (you may reference [this example repository](https://gitlab.com/yookoala/code-coverage-visualization-with-php/)), you can run the test and -generate the coverage xml: +generate the `coverage.xml`: ```yaml run tests: @@ -283,7 +290,9 @@ run tests: - php ./vendor/bin/phpunit --coverage-text --coverage-cobertura=coverage.cobertura.xml artifacts: reports: - cobertura: coverage.cobertura.xml + coverage_report: + coverage_format: cobertura + path: coverage.cobertura.xml ``` [Codeception](https://codeception.com/), through PHPUnit, also supports generating Cobertura report with @@ -318,7 +327,9 @@ run tests: name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA} expire_in: 2 days reports: - cobertura: build/coverage.xml + coverage_report: + coverage_format: cobertura + path: build/coverage.xml ``` ### Go example @@ -345,7 +356,9 @@ run tests: - go run github.com/boumenot/gocover-cobertura < coverage.txt > coverage.xml artifacts: reports: - cobertura: coverage.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml ``` ### Ruby example @@ -372,5 +385,7 @@ run tests: - bundle exec rspec artifacts: reports: - cobertura: coverage/coverage.xml + coverage_report: + coverage_format: cobertura + path: coverage/coverage.xml ``` diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 4501cf500b0..c2b85a2183c 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -53,10 +53,14 @@ If you're in a project and select **Issues > Milestones**, GitLab displays only ## Creating milestones -Users with at least the Developer role can create milestones. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. Milestones can be created either at project or group level. +Prerequisites: + +- You must have at least the Reporter role for a group. + To create a milestone: 1. On the top bar, select **Menu > Projects** and find your project or **Menu > Groups** and find your group. @@ -69,7 +73,13 @@ To create a milestone: ## Editing milestones -Users with at least the Developer role can edit milestones. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/343889) the minimum user role from Developer to Reporter in GitLab 15.0. + +Users with at least the Reporter role can edit milestones. + +Prerequisites: + +- You must have at least the Reporter role for a group. To edit a milestone: diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md index f08afc924f3..5fd17b5c07e 100644 --- a/doc/user/project/pages/getting_started/pages_from_scratch.md +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -202,7 +202,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH pages: script: @@ -222,7 +222,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH pages: script: @@ -233,7 +233,7 @@ pages: paths: - public rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: $CI_COMMIT_BRANCH == "main" ``` ### Specify a stage to deploy @@ -253,7 +253,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH pages: stage: deploy @@ -265,7 +265,7 @@ pages: paths: - public rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: $CI_COMMIT_BRANCH == "main" ``` Now add another job to the CI file, telling it to @@ -276,7 +276,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH pages: stage: deploy @@ -288,7 +288,7 @@ pages: paths: - public rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: $CI_COMMIT_BRANCH == "main" test: stage: test @@ -300,7 +300,7 @@ test: paths: - test rules: - - if: '$CI_COMMIT_BRANCH != "main"' + - if: $CI_COMMIT_BRANCH != "main" ``` When the `test` job runs in the `test` stage, Jekyll @@ -327,7 +327,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH before_script: - gem install bundler @@ -341,7 +341,7 @@ pages: paths: - public rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: $CI_COMMIT_BRANCH == "main" test: stage: test @@ -351,7 +351,7 @@ test: paths: - test rules: - - if: '$CI_COMMIT_BRANCH != "main"' + - if: $CI_COMMIT_BRANCH != "main" ``` ### Build faster with cached dependencies @@ -367,7 +367,7 @@ image: ruby:2.7 workflow: rules: - - if: '$CI_COMMIT_BRANCH' + - if: $CI_COMMIT_BRANCH cache: paths: @@ -385,7 +385,7 @@ pages: paths: - public rules: - - if: '$CI_COMMIT_BRANCH == "main"' + - if: $CI_COMMIT_BRANCH == "main" test: stage: test @@ -395,7 +395,7 @@ test: paths: - test rules: - - if: '$CI_COMMIT_BRANCH != "main"' + - if: $CI_COMMIT_BRANCH != "main" ``` In this case, you need to exclude the `/vendor` diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index f274338c2fd..5846ceeb1b6 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -304,7 +304,7 @@ Find more details in the [Pages administration documentation](../../../administr Safari requires the web server to support the [Range request header](https://developer.apple.com/library/archive/documentation/AppleApplications/Reference/SafariWebContent/CreatingVideoforSafarioniPhone/CreatingVideoforSafarioniPhone.html#//apple_ref/doc/uid/TP40006514-SW6) in order to play your media content. For GitLab Pages to serve -HTTP Range requests, you should use the following two variables in your `.gitlab-ci.yaml` file: +HTTP Range requests, you should use the following two variables in your `.gitlab-ci.yml` file: ```yaml pages: diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index 06396b5cd62..56f5a2d24ff 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -10,7 +10,7 @@ In GitLab, [permissions](../permissions.md) are fundamentally defined around the idea of having read or write permission to the repository and branches. To impose further restrictions on certain branches, they can be protected. -The default branch for your repository is protected by default. +The [default branch](repository/branches/default.md) for your repository is protected by default. ## Who can modify a protected branch @@ -39,7 +39,8 @@ Prerequisite: To protect a branch: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to merge** list, select a role, or group that can merge into this branch. In GitLab Premium, you can also add users. @@ -50,13 +51,18 @@ The protected branch displays in the list of protected branches. ## Configure multiple protected branches by using a wildcard +If both a specific rule and a wildcard rule apply to the same branch, the most +permissive rule controls how the branch behaves. For merge controls to work properly, +set **Allowed to push** to a broader set of users than **Allowed to merge**. + Prerequisite: - You must have at least the Maintainer role. To protect multiple branches at the same time: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, type the branch name and a wildcard. For example: @@ -88,7 +94,8 @@ from the command line or from a Git client application. To create a new branch through the user interface: -1. Go to **Repository > Branches**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Repository > Branches**. 1. Select **New branch**. 1. Fill in the branch name and select an existing branch, tag, or commit to base the new branch on. Only existing protected branches and commits @@ -96,21 +103,28 @@ To create a new branch through the user interface: ## Require everyone to submit merge requests for a protected branch -You can force everyone to submit a merge request, rather than allowing them to check in directly -to a protected branch. This is compatible with workflows like the [GitLab workflow](../../topics/gitlab_flow.md). +You can force everyone to submit a merge request, rather than allowing them to +check in directly to a protected branch. This setting is compatible with workflows +like the [GitLab workflow](../../topics/gitlab_flow.md). -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to merge** list, select **Developers + Maintainers**. 1. From the **Allowed to push** list, select **No one**. + + NOTE: + Setting a role, group or user as **Allowed to push** also allows those users to merge. + 1. Select **Protect**. ## Allow everyone to push directly to a protected branch You can allow everyone with write access to push to the protected branch. -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to push** list, select **Developers + Maintainers**. @@ -137,7 +151,8 @@ Prerequisites: To allow a deploy key to push to a protected branch: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to push** list, select the deploy key. @@ -155,7 +170,8 @@ protected branches. To protect a new branch and enable force push: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to push** and **Allowed to merge** lists, select the settings you want. @@ -166,7 +182,8 @@ To protect a new branch and enable force push: To enable force pushes on branches that are already protected: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. In the list of protected branches, next to the branch, turn on the **Allowed to force push** toggle. @@ -181,7 +198,8 @@ For a protected branch, you can require at least one approval by a [Code Owner]( To protect a new branch and enable Code Owner's approval: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to push** and **Allowed to merge** lists, select the settings you want. @@ -190,7 +208,8 @@ To protect a new branch and enable Code Owner's approval: To enable Code Owner's approval on branches that are already protected: -1. Go to your project and select **Settings > Repository**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. In the list of protected branches, next to the branch, turn on the **Code owner approval** toggle. @@ -221,9 +240,11 @@ for details about the pipelines security model. Users with at least the Maintainer role can manually delete protected branches by using the GitLab web interface: -1. Go to **Repository > Branches**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Repository > Branches**. 1. Next to the branch you want to delete, select **Delete** (**{remove}**). -1. On the confirmation dialog, type the branch name and select **Delete protected branch**. +1. On the confirmation dialog, type the branch name. +1. Select **Yes, delete protected branch**. Protected branches can only be deleted by using GitLab either from the UI or API. This prevents accidentally deleting a branch through local Git commands or diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index d04f79d11c7..6ef8477b6b6 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -63,6 +63,7 @@ time as pushing changes: | `merge_request.remove_source_branch` | Set the merge request to remove the source branch when it's merged. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | | `merge_request.title="<title>"` | Set the title of the merge request. Ex: `git push -o merge_request.title="The title I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | | `merge_request.description="<description>"` | Set the description of the merge request. Ex: `git push -o merge_request.description="The description I want"`. | [12.2](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64320) | +| `merge_request.draft` | Mark the merge request as a draft. Ex: `git push -o merge_request.draft`. | [15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/296673) | | `merge_request.milestone="<milestone>"` | Set the milestone of the merge request. Ex: `git push -o merge_request.milestone="3.0"`. | [14.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63960) | | `merge_request.label="<label>"` | Add labels to the merge request. If the label does not exist, it is created. For example, for two labels: `git push -o merge_request.label="label1" -o merge_request.label="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | | `merge_request.unlabel="<label>"` | Remove labels from the merge request. For example, for two labels: `git push -o merge_request.unlabel="label1" -o merge_request.unlabel="label2"`. | [12.3](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31831) | diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 26b36198bde..c0a6fa9c301 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -91,6 +91,7 @@ To create a release in the Releases page: - [Title](#title). - [Milestones](#associate-milestones-with-a-release). - [Release notes](#release-notes-description). + - Whether or not to include the [Tag message](../../../topics/git/tags.md). - [Asset links](#links). 1. Select **Create release**. @@ -439,8 +440,11 @@ Every release has a description. You can add any text you like, but we recommend including a changelog to describe the content of your release. This helps users quickly scan the differences between each release you publish. -[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) and -Release note descriptions are unrelated. Description supports [Markdown](../../markdown.md). +[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can +be included in Release note descriptions by selecting **Include tag message in +the release notes**. + +Description supports [Markdown](../../markdown.md). ### Release assets @@ -529,7 +533,7 @@ The physical location of the asset can change at any time and the direct link re > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. -The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanant URL to download an asset from the *latest release*. +The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*. The format of the URL is: diff --git a/doc/user/project/repository/gpg_signed_commits/index.md b/doc/user/project/repository/gpg_signed_commits/index.md index 00998c9a227..b2a6c8848ce 100644 --- a/doc/user/project/repository/gpg_signed_commits/index.md +++ b/doc/user/project/repository/gpg_signed_commits/index.md @@ -11,7 +11,7 @@ GPG ([GNU Privacy Guard](https://gnupg.org/)) key. When you add a cryptographic signature to your commit, you provide extra assurance that a commit originated from you, rather than an impersonator. If GitLab can verify a commit author's identity with a public GPG key, the commit is marked **Verified** in the -GitLab UI. You can then configure [push rules](../push_rules.md#enabling-push-rules) +GitLab UI. You can then configure [push rules](../push_rules.md) for your project to reject individual commits not signed with GPG, or reject all commits from unverified users. diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index 6ece6e3e4e0..02b5639cae8 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -244,6 +244,11 @@ When you [rename a user](../../profile/index.md#change-your-username), - The redirects are available as long as the original path is not claimed by another group, user, or project. +WARNING: +The [CI/CD `includes` keyword](../../../ci/yaml/includes.md) can't follow project +redirects. Pipelines fail with a syntax error when configured to use `includes` +to fetch configuration from a project that is renamed or moved. + ## Related topics - [GitLab Workflow VS Code extension](vscode.md). diff --git a/doc/user/project/repository/jupyter_notebooks/index.md b/doc/user/project/repository/jupyter_notebooks/index.md index 39b57f89ceb..d013d2802bf 100644 --- a/doc/user/project/repository/jupyter_notebooks/index.md +++ b/doc/user/project/repository/jupyter_notebooks/index.md @@ -23,21 +23,25 @@ it's rendered into HTML when you view it: Interactive features, including JavaScript plots, don't work when viewed in GitLab. -## Cleaner diffs +## Cleaner diffs and raw diffs > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6589) in GitLab 14.5 as an [Alpha](../../../../policy/alpha-beta-support.md#alpha-features) release [with a flag](../../../../administration/feature_flags.md) named `jupyter_clean_diffs`. Enabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75500) in GitLab 14.9. Feature flag `jupyter_clean_diffs` removed. > - [Reintroduced toggle](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85079) in GitLab 15.0 [with a flag](../../../../administration/feature_flags.md) named `ipynb_semantic_diff`. Enabled by default. +> - Selecting between raw and cleaner diffs [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85203) in GitLab 15.0 [with a flag](../../../../administration/feature_flags.md) named `rendered_diffs_viewer`. Enabled by default. FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md) named `ipynb_semantic_diff`. +On self-managed GitLab, by default semantic diffs are available. To hide the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md) named `ipynb_semantic_diff`. On GitLab.com, this feature is available. -This feature is ready for production use. + +FLAG: +On self-managed GitLab, by default the ability to switch between raw and rendered diffs is available. To hide the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md)named `rendered_diffs_viewer`. On GitLab.com, this feature is available. When commits include changes to Jupyter Notebook files, GitLab: - Transforms the machine-readable `.ipynb` file into a human-readable Markdown file. - Displays a cleaner version of the diff that includes syntax highlighting. +- Enables switching between raw and rendered diffs on the Commit and Compare pages. (Not available on merge request pages.) Code suggestions are not available on diffs and merge requests for `.ipynb` files. diff --git a/doc/user/project/repository/managing_large_repositories.md b/doc/user/project/repository/managing_large_repositories.md new file mode 100644 index 00000000000..d6f88697c54 --- /dev/null +++ b/doc/user/project/repository/managing_large_repositories.md @@ -0,0 +1,51 @@ +--- +stage: Enablement +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +description: "Documentation on large repositories." +--- + +# Managing large repositories **(FREE SELF)** + +GitLab, like any Git based system, is subject to similar performance restraints when it comes to large +repositories that size into the gigabytes. + +On this page we detail several best practices to improve performance with these large repositories on GitLab. + +## Large File System (LFS) + +It's *strongly* recommended in any Git system that binary or blob files (for example, packages, audio, video, graphics, etc.) are stored as Large File Storage (LFS) objects. In such setup, the Objects are stored elsewhere, such as in Object Storage, and this can reduce the repository size significantly, thus improving performance. + +Refer to the [Git LFS docs for more info](../../../topics/git/lfs/index.md). + +## Gitaly Pack Objects Cache + +Gitaly, the service that provides storage for Git repositories, can be configured to cache a short rolling window of Git fetch responses. This is recommended for large repositories as it can notably reduce server load when your server receives lots of fetch traffic. + +Refer to the [Gitaly Pack Objects Cache for more info](../../../administration/gitaly/configure_gitaly.md#pack-objects-cache). + +## Reference Architectures + +Large repositories tend to be found in larger organisations with many users. The GitLab Quality and Support teams provide several [Reference Architectures](../../../administration/reference_architectures/index.md) that are the recommended way to deploy GitLab at scale. + +In these types of setups it's recommended that the GitLab environment used matches a Reference Architecture to improve performance. + +## Gitaly Cluster + +Gitaly Cluster can notably improve large repository performance as it holds multiple replicas of the repository across several nodes. As a result, Gitaly Cluster can load balance read requests against those repositories and is also fault tolerant. + +It's recommended for large repositories, however, Gitaly Cluster is a large solution with additional complexity of setup and management. Refer to the [Gitaly Cluster docs for more info](../../../administration/gitaly/index.md), specifically the [Before deploying Gitaly Cluster](../../../administration/gitaly/index.md#before-deploying-gitaly-cluster) section. + +## Keep GitLab up to date + +Performance improvements and fixes are added continuously in GitLab. As such, it's recommended you keep GitLab updated to the latest version where possible to benefit from these. + +## Reduce concurrent clones in CI/CD + +Large repositories tend to be monorepos. This in turn typically means that these repositories get a lot of traffic not only from users, but from CI/CD. + +CI/CD loads tend to be concurrent as pipelines are scheduled during set times. As a result, the Git requests against the repositories can spike notably during these times and lead to reduced performance for both CI and users alike. + +When designing CI/CD pipelines, it's advisable to reduce their concurrency by staggering them to run at different times, for example, a set running at one time and then another set running several minutes later. + +There's several other actions that can be explored to improve CI/CD performance with large repositories. Refer to the [Runner docs for more info](../../../ci/large_repositories/index.md). diff --git a/doc/user/project/repository/mirror/index.md b/doc/user/project/repository/mirror/index.md index e1017e78437..1645cf7244e 100644 --- a/doc/user/project/repository/mirror/index.md +++ b/doc/user/project/repository/mirror/index.md @@ -228,10 +228,19 @@ This error can occur when a firewall performs a `Deep SSH Inspection` on outgoin If you receive this error after creating a new project using [GitLab CI/CD for external repositories](../../../../ci/ci_cd_for_external_repos/): -```plaintext -"2:fetch remote: "fatal: could not read Username for 'https://bitbucket.org': -terminal prompts disabled\n": exit status 128." -``` +- In Bitbucket Cloud: + + ```plaintext + "2:fetch remote: "fatal: could not read Username for 'https://bitbucket.org': + terminal prompts disabled\n": exit status 128." + ``` + +- In Bitbucket Server (self-managed): + + ```plaintext + "2:fetch remote: "fatal: could not read Username for 'https://lab.example.com': + terminal prompts disabled\n": exit status 128. + ``` Check if the repository owner is specified in the URL of your mirrored repository: @@ -239,13 +248,21 @@ Check if the repository owner is specified in the URL of your mirrored repositor 1. On the left sidebar, select **Settings > Repository**. 1. Expand **Mirroring repositories**. 1. If no repository owner is specified, delete and add the URL again in this format, - replacing `OWNER`, `ACCOUNTNAME`, and `REPONAME` with your values: + replacing `OWNER`, `ACCOUNTNAME`, `PATH_TO_REPO`, and `REPONAME` with your values: - ```plaintext - https://OWNER@bitbucket.org/ACCOUNTNAME/REPONAME.git - ``` + - In Bitbucket Cloud: + + ```plaintext + https://OWNER@bitbucket.org/ACCOUNTNAME/REPONAME.git + ``` + + - In Bitbucket Server (self-managed): + + ```plaintext + https://OWNER@lab.example.com/PATH_TO_REPO/REPONAME.git + ``` -When connecting to the repository for mirroring, Bitbucket requires the repository owner in the string. +When connecting to the Cloud or self-managed Bitbucket repository for mirroring, the repository owner is required in the string. ### Pull mirror is missing LFS files @@ -257,3 +274,31 @@ In some cases, pull mirroring does not transfer LFS files. This issue occurs whe [Fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/335123) in GitLab 14.0.6. - You mirror an external repository using object storage. An issue exists [to fix this problem](https://gitlab.com/gitlab-org/gitlab/-/issues/335495). + +### `The repository is being updated`, but neither fails nor succeeds visibly + +In rare cases, mirroring slots on Redis can become exhausted, +possibly because Sidekiq workers are reaped due to out-of-memory (OoM) events. +When this occurs, mirroring jobs start and complete quickly, but they neither +fail nor succeed. They also do not leave a clear log. To check for this problem: + +1. Enter the [Rails console](../../../../administration/operations/rails_console.md) + and check Redis' mirroring capacity: + + ```ruby + current = Gitlab::Redis::SharedState.with { |redis| redis.scard('MIRROR_PULL_CAPACITY') }.to_i + maximum = Gitlab::CurrentSettings.mirror_max_capacity + available = maximum - current + ``` + +1. If the mirroring capacity is `0` or very low, you can drain all stuck jobs with: + + ```ruby + Gitlab::Redis::SharedState.with { |redis| redis.smembers('MIRROR_PULL_CAPACITY') }.each do |pid| + Gitlab::Redis::SharedState.with { |redis| redis.srem('MIRROR_PULL_CAPACITY', pid) } + end + ``` + +1. After you run the command, the [background jobs page](../../../admin_area/index.md#background-jobs) + should show new mirroring jobs being scheduled, especially when + [triggered manually](#update-a-mirror). diff --git a/doc/user/project/repository/mirror/pull.md b/doc/user/project/repository/mirror/pull.md index 4c437d0f8c8..73103a9af3d 100644 --- a/doc/user/project/repository/mirror/pull.md +++ b/doc/user/project/repository/mirror/pull.md @@ -62,7 +62,8 @@ Prerequisite: 1. Enter the **Git repository URL**. Include the username in the URL, if required: `https://MYUSERNAME@github.com/GROUPNAME/PROJECTNAME.git` 1. In **Mirror direction**, select **Pull**. -1. In **Authentication method**, select your authentication method. +1. In **Authentication method**, select your authentication method. To learn more, read + [Authentication methods for mirrors](index.md#authentication-methods-for-mirrors). 1. Select any of the options you need: - [**Overwrite diverged branches**](#overwrite-diverged-branches) - [**Trigger pipelines for mirror updates**](#trigger-pipelines-for-mirror-updates) @@ -114,6 +115,21 @@ and mirroring attempts stop. This failure is visible in either the: To resume project mirroring, [force an update](index.md#force-an-update). +If many projects are affected by this problem, such as after a long network or +server outage, you can use the [Rails console](../../../../administration/operations/rails_console.md) +to identify and update all affected projects with this command: + +```ruby +Project.find_each do |p| + if p.import_state && p.import_state.retry_count >= 14 + puts "Resetting mirroring operation for #{p.full_path}" + p.import_state.reset_retry_count + p.import_state.set_next_execution_to_now(prioritized: true) + p.import_state.save! + end +end +``` + ## Related topics - Configure [pull mirroring intervals](../../../../administration/instance_limits.md#pull-mirroring-interval) diff --git a/doc/user/project/repository/mirror/push.md b/doc/user/project/repository/mirror/push.md index ebc4430ac53..c00ebf415c9 100644 --- a/doc/user/project/repository/mirror/push.md +++ b/doc/user/project/repository/mirror/push.md @@ -38,8 +38,8 @@ To set up push mirroring for an existing project: 1. Expand **Mirroring repositories**. 1. Enter a repository URL. 1. In the **Mirror direction** dropdown list, select **Push**. -1. Select an **Authentication method**. - You can authenticate with either a password or an [SSH key](index.md#ssh-authentication). +1. Select an **Authentication method**. To learn more, read + [Authentication methods for mirrors](index.md#authentication-methods-for-mirrors). 1. Select **Only mirror protected branches**, if necessary. 1. Select **Keep divergent refs**, if desired. 1. To save the configuration, select **Mirror repository**. diff --git a/doc/user/project/repository/push_rules.md b/doc/user/project/repository/push_rules.md index bb473a2830b..994cf78d98a 100644 --- a/doc/user/project/repository/push_rules.md +++ b/doc/user/project/repository/push_rules.md @@ -6,148 +6,136 @@ info: "To determine the technical writer assigned to the Stage/Group associated # Push rules **(PREMIUM)** -Gain additional control over what can and can't be pushed to your repository by using -regular expressions to reject pushes based on commit contents, branch names or file details. - -GitLab already offers [protected branches](../protected_branches.md), but there are -cases when you need some specific rules. Some common scenarios: preventing Git tag removal, or -enforcing a special format for commit messages. - Push rules are [pre-receive Git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) you -can enable in a user-friendly interface. They are defined either: +can enable in a user-friendly interface. Push rules give you more control over what +can and can't be pushed to your repository. While GitLab offers +[protected branches](../protected_branches.md), you may need more specific rules, such as: -- Globally if you are an administrator. -- Per project, so you can have different rules applied to different - projects depending on your needs. +- Evaluating the contents of a commit. +- Confirming commit messages match expected formats. +- Enforcing branch name rules. +- Evaluating the details of files. +- Preventing Git tag removal. -## Use cases +GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions +in push rules. You can test them at the [regex101 regex tester](https://regex101.com/). -Every push rule could have its own use case, but let's consider some examples. +For custom push rules use [server hooks](../../../administration/server_hooks.md). -### Commit messages with a specific reference +## Enable global push rules -Let's assume you have the following requirements for your workflow: +You can create push rules for all new projects to inherit, but they can be overridden +at the project level or the [group level](../../group/index.md#group-push-rules). -- every commit should reference a Jira issue, for example: `Refactored css. Fixes JIRA-123.` -- users should not be able to remove Git tags with `git push` +Prerequisite: -Write a regular expression that requires the mention -of a Jira issue in the commit message, like `JIRA\-\d+`. +- You must be an administrator. -Now when a user tries to push a commit with a message `Bugfix`, their push is -declined. Only pushing commits with messages like `Bugfix according to JIRA-123` -is accepted. +To create global push rules: -### Restrict branch names +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Push Rules**. +1. Expand **Push rules**. +1. Set the rule you want. +1. Select **Save push rules**. -If your company has a strict policy for branch names, you may want the branches to start -with a certain name. This approach enables different -GitLab CI/CD jobs (such as `feature`, `hotfix`, `docker`, `android`) that rely on the -branch name. +## Override global push rules per project -Your developers may not remember that policy, so they might push to -various branches, and CI pipelines might not work as expected. By restricting the -branch names globally in Push Rules, such mistakes are prevented. -All branch names that don't match your push rule are rejected. +The push rule of an individual project overrides the global push rule. +To override global push rules for a specific project: -Note that the name of your default branch is always allowed, regardless of the branch naming -regular expression (regex) specified. GitLab is configured this way -because merges typically have the default branch as their target. -If you have other target branches, include them in your regex. (See [Enabling push rules](#enabling-push-rules)). +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Repository**. +1. Expand **Push rules**. +1. Set the rule you want. +1. Select **Save push rules**. -The default branch also defaults to being a [protected branch](../protected_branches.md), -which already limits users from pushing directly. +## Verify users -Some example regular expressions you can use in push rules: +Use these rules to validate users who make commits. -- `^JIRA-` Branches must start with `JIRA-`. -- `-JIRA$` Branches must end with `-JIRA`. -- `^[a-z0-9\\-]{4,15}$` Branches must be between `4` and `15` characters long, - accepting only lowercase letters, numbers and dashes. +- **Reject unverified users**: Users must have a [confirmed email address](../../../security/user_email_confirmation.md). +- **Check whether the commit author is a GitLab user**: The commit author and committer must have an email address that's been verified by GitLab. +- **Commit author's email**: Both the author's and committer's email addresses must match the regular expression. + To allow any email address, leave empty. -#### Default restricted branch names +## Validate commit messages -> Introduced in GitLab 12.10. +Use these rules for your commit messages. -By default, GitLab restricts certain formats of branch names for security purposes. -40-character hexadecimal names, similar to Git commit hashes, are prohibited. +- **Require expression in commit messages**: Messages must match the + expression. To allow any commit message, leave empty. + Uses multiline mode, which can be disabled by using `(?-m)`. -### Custom Push Rules **(PREMIUM SELF)** + For example, if every commit should reference a Jira issue + (like `Refactored css. Fixes JIRA-123.`), the regular expression would be + `JIRA\-\d+`. +- **Reject expression in commit messages**: Commit messages must not match + the expression. To allow any commit message, leave empty. + Uses multiline mode, which can be disabled by using `(?-m)`. -It's possible to create custom push rules rather than the push rules available in -**Admin Area > Push Rules** by using more advanced server hooks. +## Validate branch names -See [server hooks](../../../administration/server_hooks.md) for more information. +To validate your branch names, enter a regular expression for **Branch name**. +To allow any branch name, leave empty. Your +[default branch](branches/default.md) is always allowed. -## Enabling push rules +Examples: -You can create push rules for all new projects to inherit, but they can be overridden -at the project level or the [group level](../../group/index.md#group-push-rules). +- Branches must start with `JIRA-`. -To create global push rules: + ```plaintext + `^JIRA-` + ``` -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Push Rules**. +- Branches must end with `-JIRA`. -To override global push rules in a project's settings: + ```plaintext + `-JIRA$` + ``` -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Repository**. -1. Expand **Push rules**. -1. Set the rule you want. -1. Select **Save push rules**. +- Branches must be between `4` and `15` characters long, + accepting only lowercase letters, numbers and dashes. -The following options are available: + ```plaintext + `^[a-z0-9\\-]{4,15}$` + ``` -| Push rule | Description | -|---------------------------------|-------------| -| Removal of tags with `git push` | Forbid users to remove Git tags with `git push`. Tags can be deleted through the web UI. | -| Check whether the commit author is a GitLab user | Restrict commits to existing GitLab users (checked against their emails). <sup>1</sup> | -| Reject unverified users | GitLab rejects any commit that was not committed by the same user as the user who pushed it, or where the committer's email address is not [confirmed](../../../security/user_email_confirmation.md). | -| Check whether commit is signed through GPG | Reject commit when it is not signed through GPG. Read [signing commits with GPG](gpg_signed_commits/index.md). | -| Prevent pushing secret files | GitLab rejects any files that are likely to contain secrets. See the [forbidden file names](#prevent-pushing-secrets-to-the-repository). | -| Require expression in commit messages | Only commit messages that match this regular expression are allowed to be pushed. <sup>2</sup> Leave empty to allow any commit message. Uses multiline mode, which can be disabled using `(?-m)`. | -| Reject expression in commit messages | Only commit messages that do not match this regular expression are allowed to be pushed. <sup>2</sup> Leave empty to allow any commit message. Uses multiline mode, which can be disabled using `(?-m)`. | -| Restrict by branch name | Only branch names that match this regular expression are allowed to be pushed. <sup>2</sup> Leave empty to allow all branch names. | -| Restrict by commit author's email | Only commit author's email that match this regular expression are allowed to be pushed. <sup>1</sup> <sup>2</sup> Leave empty to allow any email. | -| Prohibited file names | Any committed filenames that match this regular expression and do not already exist in the repository are not allowed to be pushed. <sup>2</sup> Leave empty to allow any filenames. See [common examples](#prohibited-file-names). | -| Maximum file size | Pushes that contain added or updated files that exceed this file size (in MB) are rejected. Set to 0 to allow files of any size. Files tracked by Git LFS are exempted. | +NOTE: +In GitLab 12.10 and later, certain formats of branch names are restricted by +default for security purposes. 40-character hexadecimal names, similar to Git +commit hashes, are prohibited. -1. Checks both the commit author and committer. -1. GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions in push rules, and you can test them at the [regex101 regex tester](https://regex101.com/). +## Prevent unintended consequences -### Caveat to "Reject unsigned commits" push rule +Use these rules to prevent unintended consequences. -This push rule ignores commits that are authenticated and created by GitLab -(either through the UI or API). When the **Reject unsigned commits** push rule is -enabled, unsigned commits may still show up in the commit history if a commit was -created **in** GitLab itself. As expected, commits created outside GitLab and -pushed to the repository are rejected. For more information about how GitLab -plans to fix this issue, read [issue #19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). +- **Reject unsigned commits**: Commit must be signed through [GPG](gpg_signed_commits/index.md). This rule + can block some legitimate commits [created in the Web IDE](#reject-unsigned-commits-push-rule-disables-web-ide), + and allow [unsigned commits created in the GitLab UI](#unsigned-commits-created-in-the-gitlab-ui). +- **Do not allow users to remove Git tags with `git push`**: Users cannot use `git push` to remove Git tags. + Users can still delete tags in the UI. -#### "Reject unsigned commits" push rule disables Web IDE +## Validate files -In 13.10, if a project has the "Reject unsigned commits" push rule, the user is not allowed to -commit through GitLab Web IDE. +Use these rules to validate files contained in the commit. -To allow committing through the Web IDE on a project with this push rule, a GitLab administrator -must disable the feature flag `reject_unsigned_commits_by_gitlab`. This can be done through a -[rails console](../../../administration/operations/rails_console.md) and running: +- **Prevent pushing secret files**: Files must not contain [secrets](#prevent-pushing-secrets-to-the-repository). +- **Prohibited file names**: Files that do not exist in the repository + must not match the regular expression. To allow all file names, leave empty. See [common examples](#prohibit-files-by-name). +- **Maximum file size**: Added or updated files must not exceed this + file size (in MB). To allow files of any size, set to `0`. Files tracked by Git LFS are exempted. -```ruby -Feature.disable(:reject_unsigned_commits_by_gitlab) -``` - -## Prevent pushing secrets to the repository +### Prevent pushing secrets to the repository > Moved to GitLab Premium in 13.9. -Secrets, such as credential files and SSH private keys, should never be committed to a version control +Never commit secrets, such as credential files and SSH private keys, to a version control system. In GitLab, you can use a predefined list of files to block those files from a -repository. Any merge request containing a file matching the list is blocked from being merged. -Files already committed to the repository are not restricted by this push rule. +repository. Merge requests that contain a file that matches the list are blocked. +This push rule does not restrict files already committed to the repository. -Files blocked by this rule are listed below. For a complete list of criteria, see +Files blocked by this rule are listed below. For a complete list of criteria, refer to [`files_denylist.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml). - AWS CLI credential blobs: @@ -211,78 +199,85 @@ Files blocked by this rule are listed below. For a complete list of criteria, se - `*.history` - `*_history` -### Prevent pushing secrets to all projects +### Prohibit files by name -To set a global push rule to prevent pushing secrets to all projects: +> Moved to GitLab Premium in 13.9. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Push Rules**. -1. Expand **Push rules**. -1. Select **Prevent pushing secret files**. -1. Select **Save push rules**. +In Git, filenames include both the file's name, and all directories preceding the name. +When you `git push`, each filename in the push is compared to the regular expression +in **Prohibited file names**. -### Prevent pushing secrets to a project +The regular expression in your **Prohibited file names** push rule can contain multiple, +independent matches to exclude. You can match file names broadly to any location in +your repository, or restrict only in certain locations. Filename matches can also +be partial, and exclude file types by extension. -The push rule of a project overrides the global push rule. +These examples use regex (regular expressions) string boundary characters to match +the beginning of a string (`^`), and its end (`$`). They also include instances +where either the directory path or the filename can include `.` or `/`. Both of +these special regex characters must be escaped with a backslash `\\` if you want +to use them as normal characters in a match condition. -To prevent pushing secrets to a project: +- **Prevent pushing `.exe` files to any location in the repository** - This regex + matches any filename that contains `.exe` at the end: -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Repository**. -1. Expand **Push rules**. -1. Select **Prevent pushing secret files**. -1. Select **Save push rules**. + ```plaintext + \.exe$ + ``` -## Prohibited file names +- **Prevent pushing a specific configuration file in the repository root** -> Moved to GitLab Premium in 13.9. + ```plaintext + ^config\.yml$ + ``` -Each filename contained in a Git push is compared to the regular expression in this field. Filenames in Git consist of both the file's name and any directory that may precede it. A singular regular expression can contain multiple independent matches used as exclusions. File names can be broadly matched to any location in the repository, or restricted to specific locations. Filenames can also be partial matches used to exclude file types by extension. +- **Prevent pushing a specific configuration file in a known directory** -The following examples make use of regex string boundary characters which match the beginning of a string (`^`), and the end (`$`). They also include instances where either the directory path or the filename can include `.` or `/`. Both of these special regex characters have to be escaped with a backslash `\\` to be used as normal characters in a match condition. + ```plaintext + ^directory-name\/config\.yml$ + ``` -Example: prevent pushing any `.exe` files to any location in the repository. This uses a partial match, which matches any filename that contains `.exe` at the end: +- **Prevent pushing a specific file to any location in the repository** - This example tests + for any file named `install.exe`. The parenthesized expression `(^|\/)` matches either + a file following a directory separator, or a file in the root directory of the repository: -```plaintext -\.exe$ -``` + ```plaintext + (^|\/)install\.exe$ + ``` -Example: prevent a specific configuration file in the repository root from being pushed: +- **Combine all previous expressions into one expression** - The preceding expressions rely + on the end-of-string character `$`. We can move that part of each expression to the + end of the grouped collection of match conditions, where it is appended to all matches: -```plaintext -^config\.yml$ -``` + ```plaintext + (\.exe|^config\.yml|^directory-name\/config\.yml|(^|\/)install\.exe)$ + ``` -Example: prevent a specific configuration file in a known directory from being pushed: +## Related topics -```plaintext -^directory-name\/config\.yml$ -``` +- [Server hooks](../../../administration/server_hooks.md), to create complex custom push rules +- [Signing commits with GPG](gpg_signed_commits/index.md) +- [Protected branches](../protected_branches.md) -Example: prevent the specific file named `install.exe` from being pushed to any -location in the repository. The parenthesized expression `(^|\/)` matches either -a file following a directory separator or a file in the root directory of the repository: +## Troubleshooting -```plaintext -(^|\/)install\.exe$ -``` +### Reject unsigned commits push rule disables Web IDE -Example: combining all of the above in a single expression. The preceding expressions rely -on the end-of-string character `$`. We can move that part of each expression to the -end of the grouped collection of match conditions where it is appended to all matches: +In GitLab 13.10, if a project has the **Reject unsigned commits** push rule, the user cannot +create commits through the GitLab Web IDE. -```plaintext -(\.exe|^config\.yml|^directory-name\/config\.yml|(^|\/)install\.exe)$ -``` +To allow committing through the Web IDE on a project with this push rule, a GitLab administrator +must disable the feature flag `reject_unsigned_commits_by_gitlab`. [with a flag](../../../administration/feature_flags.md) -<!-- ## Troubleshooting +```ruby +Feature.disable(:reject_unsigned_commits_by_gitlab) +``` -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. +### Unsigned commits created in the GitLab UI -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +The **Reject unsigned commits** push rule ignores commits that are authenticated +and created by GitLab (either through the UI or API). When this push rule is +enabled, unsigned commits may still appear in the commit history if a commit was +created in GitLab itself. As expected, commits created outside GitLab and +pushed to the repository are rejected. For more information about this issue, +read [issue #19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). diff --git a/doc/user/project/repository/web_editor.md b/doc/user/project/repository/web_editor.md index 6eed1717507..747bd690911 100644 --- a/doc/user/project/repository/web_editor.md +++ b/doc/user/project/repository/web_editor.md @@ -159,7 +159,7 @@ repository project, GitLab performs these actions: - Creates a default branch. - Commits a blank `README.md` file to it. - Creates and redirects you to a new branch based on the issue title. -- _If your project is [configured with a deployment service](../integrations/overview.md) like Kubernetes,_ +- _If your project is [configured with a deployment service](../integrations/index.md) like Kubernetes,_ GitLab prompts you to set up [auto deploy](../../../topics/autodevops/stages.md#auto-deploy) by helping you create a `.gitlab-ci.yml` file. diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 30261ed5082..9b37e147a52 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -41,7 +41,7 @@ Prerequisites: To export a project and its data, follow these steps: 1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings**. +1. On the left sidebar, select **Settings > General**. 1. Expand **Advanced**. 1. Select **Export project**. 1. After the export is generated, you should receive an email with a link to download the file. @@ -82,6 +82,7 @@ The following items are **not** exported: - Merge Request Approvers and [the number of required approvals](https://gitlab.com/gitlab-org/gitlab/-/issues/221088) - Repository size limits - Deploy keys allowed to push to protected branches +- Secure Files These content rules also apply to creating projects from templates on the [group](../../group/custom_project_templates.md) diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 31cda756a78..1e63472763d 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -88,11 +88,11 @@ read-only view to discourage this behavior. Compliance framework pipelines allow group owners to define a compliance pipeline in a separate repository that gets -executed in place of the local project's `gitlab-ci.yml` file. As part of this pipeline, an -`include` statement can reference the local project's `gitlab-ci.yml` file. This way, the compliance +executed in place of the local project's `.gitlab-ci.yml` file. As part of this pipeline, an +`include` statement can reference the local project's `.gitlab-ci.yml` file. This way, the compliance pipeline jobs can run alongside the project-specific jobs any time the pipeline runs. Jobs and variables defined in the compliance -pipeline can't be changed by variables in the local project's `gitlab-ci.yml` file. +pipeline can't be changed by variables in the local project's `.gitlab-ci.yml` file. When you set up the compliance framework, use the **Compliance pipeline configuration** box to link the compliance framework to specific CI/CD configuration. Use the @@ -314,7 +314,7 @@ related to the project by selecting the **Disable email notifications** checkbox Set up your project's merge request settings: -- Set up the merge request method (merge commit, [fast-forward merge](../merge_requests/fast_forward_merge.md)). +- Set up the [merge request method](../merge_requests/methods/index.md) (merge commit, fast-forward merge). - Add merge request [description templates](../description_templates.md#description-templates). - Enable [merge request approvals](../merge_requests/approvals/index.md). - Enable [status checks](../merge_requests/status_checks.md). @@ -409,10 +409,13 @@ NOTE: Only project owners and administrators have the [permissions](../../permissions.md#project-members-permissions) to transfer a project. -You can transfer an existing project into a [group](../../group/index.md). +You can transfer an existing project to another [group](../../group/index.md), +or you can transfer a [personal project](../working_with_projects.md#view-personal-projects) to a group. Prerequisites: +- A group for your project. You can [view your existing groups](../../group/index.md#view-groups) + to find a suitable group. If you don't have a group, [create one](../../group/index.md#create-a-group). - You must have at least the Maintainer role in that group. - You must be the Owner of that project. - The group to which the project is being transferred to must allow creation of new projects. @@ -423,15 +426,15 @@ Prerequisites: To transfer a project: -1. Navigate to your project's **Settings > General**. -1. Under **Advanced**, click **Expand**. -1. Under "Transfer project", choose the namespace you want to transfer the - project to. -1. Confirm the transfer by typing the project's path as instructed. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. Under **Transfer project**, choose the namespace to transfer the project to. +1. Select **Transfer project**. +1. Enter the project's name and select **Confirm**. -Once done, you are redirected to the new project's namespace. At this point, -read what happens with the -[redirects from the old project to the new one](../repository/index.md#what-happens-when-a-repository-path-changes). +You are redirected to the project's new URL. Read what happens with the +[redirects from the old URL to the new one](../repository/index.md#what-happens-when-a-repository-path-changes). NOTE: GitLab administrators can use the [administration interface](../../admin_area/index.md#administering-projects) @@ -520,7 +523,8 @@ If you want to use the fork for yourself and don't need to send you can safely remove the fork relationship. WARNING: -Once removed, the fork relationship cannot be restored. You can't send merge requests to the source, and if anyone has forked your project, their fork also loses the relationship. +Once removed, you can't send merge requests to the source, and if anyone has forked your project, their fork also loses the relationship. +To restore the fork relationship, [use the API](../../../api/projects.md#create-a-forked-fromto-relation-between-existing-projects). To do so: @@ -556,10 +560,6 @@ Automatically [create](../../../operations/incident_management/incidents.md#crea Configure Error Tracking to discover and view [Sentry errors within GitLab](../../../operations/error_tracking.md). -### Jaeger tracing - -Add the URL of a Jaeger server to allow your users to [easily access the Jaeger UI from within GitLab](../../../operations/tracing.md). - ### Status Page **(ULTIMATE)** [Add Storage credentials](../../../operations/incident_management/status_page.md#sync-incidents-to-the-status-page) diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index b66913b7223..e332b74f908 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -25,7 +25,7 @@ Use a project access token to authenticate: Project access tokens are similar to [group access tokens](../../group/settings/group_access_tokens.md) and [personal access tokens](../../profile/personal_access_tokens.md). -In self-managed instances, project access tokens are subject to the same [maximum lifetime limits](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens) as personal access tokens if the limit is set. +In self-managed instances, project access tokens are subject to the same [maximum lifetime limits](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) as personal access tokens if the limit is set. You can use project access tokens: @@ -48,7 +48,7 @@ To create a project access token: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Access Tokens**. 1. Enter a name. The token name is visible to any user with permissions to view the project. -1. Optional. Enter an expiry date for the token. The token expires on that date at midnight UTC. An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-personal-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. +1. Optional. Enter an expiry date for the token. The token expires on that date at midnight UTC. An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. 1. Select a role for the token. 1. Select the [desired scopes](#scopes-for-a-project-access-token). diff --git a/doc/user/project/static_site_editor/img/edit_this_page_button_v12_10.png b/doc/user/project/static_site_editor/img/edit_this_page_button_v12_10.png Binary files differdeleted file mode 100644 index 380d96f1db9..00000000000 --- a/doc/user/project/static_site_editor/img/edit_this_page_button_v12_10.png +++ /dev/null diff --git a/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png b/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png Binary files differdeleted file mode 100644 index 89864858ed3..00000000000 --- a/doc/user/project/static_site_editor/img/front_matter_ui_v13_4.png +++ /dev/null diff --git a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png b/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png Binary files differdeleted file mode 100644 index 52776c6a290..00000000000 --- a/doc/user/project/static_site_editor/img/wysiwyg_editor_v13_3.png +++ /dev/null diff --git a/doc/user/project/static_site_editor/index.md b/doc/user/project/static_site_editor/index.md index 220623d0372..343482757f5 100644 --- a/doc/user/project/static_site_editor/index.md +++ b/doc/user/project/static_site_editor/index.md @@ -2,35 +2,15 @@ stage: Create group: Editor info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" -type: reference, how-to -description: "The static site editor enables users to edit content on static websites without prior knowledge of the underlying templating language, site architecture or Git commands." +remove_date: '2022-08-03' +redirect_to: '../web_ide/index.md' --- -# Static Site Editor **(FREE)** +# Static Site Editor (removed) **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28758) in GitLab 12.10. -> - WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214559) in GitLab 13.0. -> - Non-Markdown content blocks not editable on the WYSIWYG mode [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216836) in GitLab 13.3. -> - Formatting Markdown [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49052) in GitLab 13.7. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77246) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77246) -in GitLab 14.7, and is planned for -[removal](https://gitlab.com/groups/gitlab-org/-/epics/7351) in GitLab 14.10. -Users should instead use the [Web Editor](../repository/web_editor.md) or [Web IDE](../web_ide/index.md). [Removal instructions](#remove-the-static-site-editor) for existing projects are included on this page. - -Static Site Editor (SSE) enables users to edit content on static websites without -prior knowledge of the underlying templating language, site architecture, or -Git commands. A contributor to your project can quickly edit a Markdown page -and submit the changes for review. For example: - -- Non-technical collaborators can edit a page directly from the browser. - They don't need to know Git and the details of your project to contribute. -- Recently hired team members can quickly edit content. -- Temporary collaborators can jump from project to project and quickly edit pages instead - of having to clone or fork every single project they need to submit changes to. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77246) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/352505) in 15.0. +Use the [Web Editor](../repository/web_editor.md) or [Web IDE](../web_ide/index.md) instead. ## Remove the Static Site Editor @@ -68,235 +48,3 @@ from an existing project, remove links that point back to the editor: `/helpers/custom_helpers.rb` entirely. 1. Clean up any extraneous configuration files. 1. Commit and push your changes. - -## Requirements - -- In order use the Static Site Editor feature, your project needs to be - pre-configured with the [Static Site Editor Middleman template](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman). -- You need to be logged into GitLab and be a member of the - project (with Developer or higher permission levels). - -## How it works - -The Static Site Editor is in an early stage of development and only supports -Middleman sites for now. You have to use a specific site template to start -using it. The project template is configured to deploy a [Middleman](https://middlemanapp.com/) -static website with [GitLab Pages](../pages/index.md). - -Once your website is up and running, an **Edit this page** button displays on -the bottom-left corner of its pages: - - - -When you click it, GitLab opens up an editor window from which the content -can be directly edited. When you're ready, you can submit your changes in a -click of a button: - - - -When an editor submits their changes, these are the following actions that GitLab -performs automatically in the background: - -1. Creates a new branch. -1. Commits their changes. - 1. Fixes formatting according to the [Handbook Markdown Style Guide](https://about.gitlab.com/handbook/markdown-guide/) - style guide and add them through another commit. -1. Opens a merge request against the default branch. - -The editor can then navigate to the merge request to assign it to a colleague for review. - -## Set up your project - -First, set up the project. Once done, you can use the Static Site Editor to -[edit your content](#edit-content). - -1. To get started, create a new project from the [Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) - template. You can either [fork it](../repository/forking_workflow.md#creating-a-fork) - or [create a new project from a template](../working_with_projects.md#create-a-project-from-a-built-in-template). -1. Edit the [`data/config.yml`](#static-site-generator-configuration) configuration file - to replace `<username>` and `<project-name>` with the proper values for - your project's path. -1. Optional. Edit the [`.gitlab/static-site-editor.yml`](#static-site-editor-configuration-file) file - to customize the behavior of the Static Site Editor. -1. When you submit your changes, GitLab triggers a CI/CD pipeline to deploy your project with GitLab Pages. -1. When the pipeline finishes, from your project's left-side menu, go to **Settings > Pages** to find the URL of your new website. -1. Visit your website and look at the bottom-left corner of the screen to see the new **Edit this page** button. - -Anyone satisfying the [requirements](#requirements) can edit the -content of the pages without prior knowledge of Git or of your site's -codebase. - -## Edit content - -> - Support for modifying the default merge request title and description [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216861) in GitLab 13.5. -> - Support for selecting a merge request template [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263252) in GitLab 13.6. - -After setting up your project, you can start editing content directly from the Static Site Editor. - -To edit a file: - -1. Visit the page you want to edit. -1. Select **Edit this page**. -1. The file is opened in the Static Site Editor in **WYSIWYG** mode. If you - wish to edit the raw Markdown instead, you can toggle the **Markdown** mode - in the bottom-right corner. -1. When you're done, click **Submit changes...**. -1. Optional. Adjust the default title and description of the merge request, to submit - with your changes. Alternatively, select a [merge request template](../../../user/project/description_templates.md#create-a-merge-request-template) - from the dropdown menu and edit it accordingly. -1. Select **Submit changes**. -1. A new merge request is automatically created and you can assign a colleague for review. - -### Text - -> Support for `*.md.erb` files [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223171) in GitLab 13.2. - -The Static Site Editors supports Markdown files (`.md`, `.md.erb`) for editing text. - -### Images - -> - Support for adding images through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216640) in GitLab 13.1. -> - Support for uploading images via the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218529) in GitLab 13.6. - -#### Upload an image - -You can upload image files via the WYSIWYG editor directly to the repository to default upload directory -`source/images`. To do so: - -1. Select the image icon (**{doc-image}**). -1. Select the **Upload file** tab. -1. To select a file from your computer, select **Choose file**. -1. Optional. Add a description to the image for SEO and accessibility ([ALT text](https://moz.com/learn/seo/alt-text)). -1. Select **Insert image**. - -The selected file can be any supported image file (`.png`, `.jpg`, `.jpeg`, `.gif`). The editor renders -thumbnail previews so you can verify the correct image is included and there aren't any references to -missing images. - -#### Link to an image - -You can also link to an image if you'd like: - -1. Select the image icon (**{doc-image}**). -1. Select the **Link to an image** tab. -1. Add the link to the image into the **Image URL** field (use the full path; relative paths are not supported yet). -1. Optional. Add a description to the image for SEO and accessibility ([ALT text](https://moz.com/learn/seo/alt-text)). -1. Select **Insert image**. - -The link can reference images already hosted in your project, an asset hosted -externally on a content delivery network, or any other external URL. The editor renders thumbnail previews -so you can verify the correct image is included and there aren't any references to missing images. - -### Videos - -> - Support for embedding YouTube videos through the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216642) in GitLab 13.5. - -You can embed YouTube videos on the WYSIWYG mode by clicking the video icon (**{live-preview}**). -The following URL/ID formats are supported: - -- **YouTube watch URLs**: `https://www.youtube.com/watch?v=0t1DgySidms` -- **YouTube embed URLs**: `https://www.youtube.com/embed/0t1DgySidms` -- **YouTube video IDs**: `0t1DgySidms` - -### Front matter - -> - Markdown front matter hidden on the WYSIWYG editor [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216834) in GitLab 13.1. -> - Ability to edit page front matter [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235921) in GitLab 13.5. - -Front matter is a flexible and convenient way to define page-specific variables in data files -intended to be parsed by a static site generator. Use it to set a page's -title, layout template, or author. You can also pass any kind of metadata to the -generator as the page renders out to HTML. Included at the very top of each data file, the -front matter is often formatted as YAML or JSON, and requires consistent and accurate syntax. - -To edit the front matter from the Static Site Editor you can use the GitLab regular file editor, -the Web IDE, or update the data directly from the WYSIWYG editor: - -1. Click the **Page settings** button on the bottom-right to reveal a web form with the data you - have on the page's front matter. The form is populated with the current data: - -  - -1. Update the values as you wish and close the panel. -1. When you're done, click **Submit changes...**. -1. Describe your changes (add a commit message). -1. Click **Submit changes**. -1. Click **View merge request** to view it. - -Adding new attributes to the page's front matter from the form is not supported. -To add new attributes: - -- Edit the file locally -- Edit the file with the GitLab regular file editor. -- Edit the file with the Web IDE. - -After adding an attribute, the form loads the new fields. - -## Configuration files - -You can customize the behavior of a project which uses the Static Site Editor with -the following configuration files: - -- The [`.gitlab/static-site-editor.yml`](#static-site-editor-configuration-file), which customizes the - behavior of the Static Site Editor. -- [Static Site Generator configuration files](#static-site-generator-configuration), - such as `data/config.yml`, which configures the Static Site Generator itself. - It also controls the **Edit this page** button when the site is generated. - -### Static Site Editor configuration file - -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4267) in GitLab 13.6. - -The `.gitlab/static-site-editor.yml` configuration file contains entries you can -use to customize behavior of the Static Site Editor (SSE). If the file does not exist, -default values which support a default Middleman project configuration are used. -The [Static Site Editor - Middleman](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman) project template generates a file pre-populated with these defaults. - -To customize the behavior of the SSE, edit `.gitlab/static-site-editor.yml`'s entries, -according to your project's needs. Make sure to respect YAML syntax. - -After the table, see an [example of the SSE configuration file](#gitlabstatic-site-editoryml-example). - -| Entry | GitLab version | Type | Default value | Description | -|---|---|---|---|---| -| `image_upload_path` | [13.6](https://gitlab.com/gitlab-org/gitlab/-/issues/216641) | String | `source/images` | Directory for images uploaded from the WYSIWYG editor. | - -#### `.gitlab/static-site-editor.yml` example - -```yaml -image_upload_path: 'source/images' # Relative path to the project's root. Don't include leading or trailing slashes. -``` - -### Static Site Generator configuration - -The Static Site Editor uses Middleman's configuration file, `data/config.yml` -to customize the behavior of the project itself. This file also controls the -**Edit this page** button, rendered through the file -[`layout.erb`](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/-/blob/master/source/layouts/layout.erb). - -To [configure the project template to your own project](#set-up-your-project), -you must replace the `<username>` and `<project-name>` in the `data/config.yml` -file with the proper values for your project's path. - -[Other Static Site Generators](#using-other-static-site-generators) used with -the Static Site Editor may use different configuration files or approaches. - -## Using Other Static Site Generators - -Although Middleman is the only Static Site Generator officially supported -by the Static Site Editor, you can configure your project's build and deployment -to use a different Static Site Generator. In this case, use the Middleman layout -as an example, and follow a similar approach to properly render an **Edit this page** -button in your Static Site Generator's layout. - -## Upgrade from GitLab 12.10 to 13.0 - -In GitLab 13.0, we [introduced breaking changes](https://gitlab.com/gitlab-org/gitlab/-/issues/213282) -to the URL structure of the Static Site Editor. Follow the instructions in this -[snippet](https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman/snippets/1976539) -to update your project with the 13.0 changes. - -## Limitations - -- The Static Site Editor still cannot be quickly added to existing Middleman sites. - Follow this [epic](https://gitlab.com/groups/gitlab-org/-/epics/2784) for updates. diff --git a/doc/user/project/time_tracking.md b/doc/user/project/time_tracking.md index 5f747d99ce7..0891e02f3f7 100644 --- a/doc/user/project/time_tracking.md +++ b/doc/user/project/time_tracking.md @@ -114,6 +114,18 @@ so if you remove more time than already entered, GitLab ignores the subtraction. To remove all the time spent at once, use the `/remove_time_spent` [quick action](quick_actions.md). +### Delete time spent + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356796) in GitLab 15.0. + +A timelog is a single entry of time spent, either positive or negative. + +Prerequisites: + +- You must be the author of the timelog or have at least the Maintainer role for the project. + +You can [delete timelogs](../../api/graphql/reference/index.md#mutationtimelogdelete) using the GraphQL API. + ## View a time tracking report > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271409) in GitLab 13.12. diff --git a/doc/user/project/web_ide/img/command_palette_v13_6.png b/doc/user/project/web_ide/img/command_palette_v13_6.png Binary files differdeleted file mode 100644 index 54580a79ebd..00000000000 --- a/doc/user/project/web_ide/img/command_palette_v13_6.png +++ /dev/null diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 8f9486633d5..9db30ee2ab6 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -23,13 +23,8 @@ and from merge requests: 1. Select **Open in Web IDE** from the list to display it as the editing option. 1. Select **Open in Web IDE** to open the editor. - *When viewing a merge request* - - 1. Go to your merge request, and select the **Overview** tab. - 1. Scroll to the widgets section, after the merge request description. - 1. Select **Open in Web IDE** if it is visible. - 1. If **Open in Web IDE** is not visible: - 1. Select the **(angle-down)** next to **Open in Gitpod**. - 1. Select **Open in Web IDE** from the list to display it as the editing option. - 1. Select **Open in Web IDE** to open the editor. + 1. Go to your merge request. + 1. In the upper right corner, select **Code**, then select **Open in Gitpod**. ## File finder @@ -52,7 +47,8 @@ Some commands have a keyboard shortcut assigned to them. The command palette displays this shortcut next to each command. You can use this shortcut to invoke the command without having to select it in the command palette. - +For a full list of keyboard shortcuts in the Web IDE, refer to the +[Keyboard shortcuts](../../shortcuts.md#web-ide) list. ## Syntax highlighting diff --git a/doc/user/project/wiki/group.md b/doc/user/project/wiki/group.md index 37f2ef8fc6a..dc448fed970 100644 --- a/doc/user/project/wiki/group.md +++ b/doc/user/project/wiki/group.md @@ -1,8 +1,7 @@ --- stage: Create group: Editor -info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" -type: reference, how-to +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Group wikis **(PREMIUM)** @@ -61,6 +60,24 @@ available, you have to: All files in the wiki are available in this Git repository. +## Configure group wiki visibility + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208412) in GitLab 15.0. + +Wikis are enabled by default in GitLab. Group [administrators](../../permissions.md) +can enable or disable a group wiki through the group settings. + +To open group settings: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Permissions and group features**. +1. Scroll to **Wiki** and select one of these options: + - **Enabled**: Everyone who can access the group can access the wiki. + - **Private**: Only group members can access the wiki. + - **Disabled**: The wiki isn't accessible, and cannot be downloaded. +1. Select **Save changes**. + ## Related topics - [Wiki settings for administrators](../../../administration/wikis/index.md) diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 7d155ea9b06..fe6c7ae62b8 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -275,7 +275,7 @@ can enable or disable a project wiki by following the instructions in Administrators for self-managed GitLab installs can [configure additional wiki settings](../../../administration/wikis/index.md). -You can't disable [group wikis](group.md) from the GitLab user interface. +You can disable group wikis from the [group settings](group.md#configure-group-wiki-visibility) ## Link an external wiki diff --git a/doc/user/project/working_with_projects.md b/doc/user/project/working_with_projects.md index 03530b59e9b..bfc83aa22f5 100644 --- a/doc/user/project/working_with_projects.md +++ b/doc/user/project/working_with_projects.md @@ -275,6 +275,18 @@ To add a star to a project: - Number of open merge requests. - Number of open issues. +## View personal projects + +Personal projects are projects created under your personal namespace. + +For example, if you create an account with the username `alex`, and create a project +called `my-project` under your username, the project is created at `https://gitlab.example.com/alex/my-project`. + +To view your personal projects: + +1. On the top bar, select **Menu > Projects > Your Projects**. +1. Under **Your projects**, select **Personal**. + ## Delete a project After you delete a project, projects in personal namespaces are deleted immediately. To delay deletion of projects in a group diff --git a/doc/user/report_abuse.md b/doc/user/report_abuse.md index c0fb29b435a..93d9a1e773e 100644 --- a/doc/user/report_abuse.md +++ b/doc/user/report_abuse.md @@ -53,10 +53,8 @@ A URL to the reported user's comment is pre-filled in the abuse report's ## Report abuse from a merge request -1. On the merge request, in the top right corner, either: - - Select **Report abuse**. This option is displayed if you do not have permission to close the merge request. - - Next to **Mark as draft**, select the down arrow (**{chevron-down}**) and then select **Report abuse**. - This option is displayed if you have permission to close the merge request. +1. On the merge request, in the top right corner, select the vertical ellipsis (**{ellipsis_v}**). +1. Select **Report abuse**. 1. Submit an abuse report. 1. Select **Send report**. diff --git a/doc/user/reserved_names.md b/doc/user/reserved_names.md index 45c5f53e33c..3cf379243e3 100644 --- a/doc/user/reserved_names.md +++ b/doc/user/reserved_names.md @@ -63,7 +63,6 @@ Currently, the following names are reserved as top level groups: - `503.html` - `admin` - `api` -- `apple-touch-icon-precomposed.png` - `apple-touch-icon.png` - `assets` - `dashboard` diff --git a/doc/user/search/global_search/advanced_search_syntax.md b/doc/user/search/global_search/advanced_search_syntax.md index 962aa00eea8..3aa016f0bff 100644 --- a/doc/user/search/global_search/advanced_search_syntax.md +++ b/doc/user/search/global_search/advanced_search_syntax.md @@ -1,49 +1,51 @@ --- stage: Enablement group: Global Search -info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" -type: reference +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Advanced Search syntax **(PREMIUM)** With [Advanced Search](../advanced_search.md), you can perform a thorough -search through your entire GitLab instance. +search of your entire GitLab instance. The Advanced Search syntax supports fuzzy or exact search queries with prefixes, -boolean operators, and much more. Advanced Search uses +boolean operators, and more. Advanced Search uses [Elasticsearch's syntax](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html#simple-query-string-syntax). WARNING: -Advanced Search searches projects' default branches only. - -See query examples on the tables below and their respective expected output. -The examples link to a search on GitLab.com to help you visualize the output. +Advanced Search searches default project branches only. ## General search -| Query example | Expected output | -|---|---| -[`“display bug”`](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=%22display+bug%22&group_id=9970&project_id=278964) | Returns the **exact phrase** _display bug_ (stemming still applies). | -[`bug -display`](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+-display&group_id=9970&project_id=278964) | Results include _bug_, and **exclude** _display_. | -[<code>bug | display</code>](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+%7C+banner&group_id=9970&project_id=278964) | Results include _bug_ **or** _display_. | -[<code>bug | (display +banner)</code>](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+%7C+%28display+%2Bbanner%29&group_id=9970&project_id=278964) | Results include _bug_ **or** _display_ **and** _banner_. | -| [`bug error 50*`](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+error+50*&group_id=9970&project_id=278964) | `*` finds **partial matches**. Results include _bug_, _error_, and the partial _50_ (looking for any 500 errors, for example). | -| [`bug \-display`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=argument+%5C-last&group_id=9970&project_id=278964) | `\` **scapes symbols**. Results include _bug_ **and** _-display_. | +<!-- markdownlint-disable --> + +| Use | Description | Example | +|-----|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `"` | Exact search | [`"gem sidekiq"`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=%22gem+sidekiq%22) | +| <code>|</code> | Or | [<code>display | banner</code>](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+%7C+banner) | +| `+` | And | [`display +banner`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=display+%2Bbanner&snippets=) | +| `-` | Exclude | [`display -banner`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+-banner) | +| `*` | Partial | [`bug error 50*`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=bug+error+50%2A&snippets=) | +| `\` | Escape | [`\*md`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=%5C*md&group_id=9970&project_id=278964) | + +## Code search -## Code Search +| Use | Description | Example | +|--------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| `filename:` | File name | [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | +| `path:` | Repository location | [`path:spec/workers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fworkers&snippets=) | +| `extension:` | File extension, without the `.` | [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | +| `blob:` | Git object ID | [`blob:998707*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707*&group_id=9970) | -| Query example | Expected output | Notes | -|---|---|---| -| [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | Returns the specified filename. | Use `*` for fuzzy matching. | -| [`path:spec/controllers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fcontrollers%2F&snippets=) | Returns the specified path location of the repository. | Use `*` for fuzzy matching. | -| [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | Returns the specified file extension. | **Do not** include a leading dot. This only works with exact matches for the extension. | -| [`blob:998707b421c89b*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707b421c89b*&group_id=9970) | Returns the specified Git object ID. | This only works with exact matches. | +`extension` and `blob` return exact matches only. -## Excluding filters +## Examples -Filters can also be inverted to filter out results from the result set by prefixing the filter name with a `-` (hyphen) character. +| Example | Description | +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------| +| [`rails -filename:gemfile.lock`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=rails+-filename%3Agemfile.lock&snippets=) | Show _rails_ in all files except the _`gemfile.lock`_ file. | +| [`RSpec.describe Resolvers -*builder`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=RSpec.describe+Resolvers+-*builder) | Show all _RSpec.describe Resolvers_ that don't start with _builder_. | +| [<code>bug | (display +banner)</code>](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+%7C+%28display+%2Bbanner%29&group_id=9970&project_id=278964) | Show _bug_ **or** _display_ **and** _banner_. | -| Query example | Expected output | -|---|---| -| [`rails -filename:gemfile.lock`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=rails+-filename%3Agemfile.lock&snippets=) | Results include _`rails`_ in all files except the _`gemfile.lock`_ file. | +<!-- markdownlint-enable --> diff --git a/doc/user/search/img/issue_search_by_id.png b/doc/user/search/img/issue_search_by_id.png Binary files differdeleted file mode 100644 index 96c0b5c31e1..00000000000 --- a/doc/user/search/img/issue_search_by_id.png +++ /dev/null diff --git a/doc/user/search/img/issue_search_by_id_v15_0.png b/doc/user/search/img/issue_search_by_id_v15_0.png Binary files differnew file mode 100644 index 00000000000..411cebc0ccb --- /dev/null +++ b/doc/user/search/img/issue_search_by_id_v15_0.png diff --git a/doc/user/search/index.md b/doc/user/search/index.md index de5f469498e..171d8a63d2d 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -108,7 +108,7 @@ You can add this URL to your feed reader. You can filter the **Issues** list to individual instances by their ID. For example, enter filter `#10` to return only issue 10. The same applies to the **Merge requests** list. Enter filter `#30` to return only merge request 30. - + ### Filtering merge requests by approvers **(PREMIUM)** diff --git a/doc/user/shortcuts.md b/doc/user/shortcuts.md index e5285d63cf4..2d753fa042a 100644 --- a/doc/user/shortcuts.md +++ b/doc/user/shortcuts.md @@ -25,21 +25,23 @@ explained in each section. These shortcuts are available in most areas of GitLab: -| Keyboard shortcut | Description | -|---------------------------------|-------------| -| <kbd>?</kbd> | Show or hide the shortcut reference sheet. | -| <kbd>Shift</kbd> + <kbd>p</kbd> | Go to your Projects page. | -| <kbd>Shift</kbd> + <kbd>g</kbd> | Go to your Groups page. | -| <kbd>Shift</kbd> + <kbd>a</kbd> | Go to your Activity page. | -| <kbd>Shift</kbd> + <kbd>l</kbd> | Go to your Milestones page. | -| <kbd>Shift</kbd> + <kbd>s</kbd> | Go to your Snippets page. | -| <kbd>s</kbd> / <kbd>/</kbd> | Put cursor in the search bar. | -| <kbd>Shift</kbd> + <kbd>i</kbd> | Go to your Issues page. | -| <kbd>Shift</kbd> + <kbd>m</kbd> | Go to your [Merge requests](project/merge_requests/index.md) page. | -| <kbd>Shift</kbd> + <kbd>t</kbd> | Go to your To-Do List page. | -| <kbd>p</kbd> then <kbd>b</kbd> | Show or hide the Performance Bar. | -| <kbd>g</kbd> then <kbd>x</kbd> | Toggle between [GitLab](https://gitlab.com/) and [GitLab Next](https://next.gitlab.com/) (GitLab SaaS only). | -| <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | +| Keyboard shortcut | Description | +|------------------------------------|-------------| +| <kbd>?</kbd> | Show or hide the shortcut reference sheet. | +| <kbd>Shift</kbd> + <kbd>p</kbd> | Go to your Projects page. | +| <kbd>Shift</kbd> + <kbd>g</kbd> | Go to your Groups page. | +| <kbd>Shift</kbd> + <kbd>a</kbd> | Go to your Activity page. | +| <kbd>Shift</kbd> + <kbd>l</kbd> | Go to your Milestones page. | +| <kbd>Shift</kbd> + <kbd>s</kbd> | Go to your Snippets page. | +| <kbd>s</kbd> / <kbd>/</kbd> | Put cursor in the search bar. | +| <kbd>f</kbd> | Put cursor in the filter bar. | +| <kbd>Shift</kbd> + <kbd>i</kbd> | Go to your Issues page. | +| <kbd>Shift</kbd> + <kbd>m</kbd> | Go to your [Merge requests](project/merge_requests/index.md) page. | +| <kbd>Shift</kbd> + <kbd>t</kbd> | Go to your To-Do List page. | +| <kbd>p</kbd>, then <kbd>b</kbd> | Show or hide the Performance Bar. | +| <kbd>Escape</kbd> | Hide tooltips or popovers. | +| <kbd>g</kbd>, then <kbd>x</kbd> | Toggle between [GitLab](https://gitlab.com/) and [GitLab Next](https://next.gitlab.com/) (GitLab SaaS only). | +| <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | Additionally, the following shortcuts are available when editing text in text fields (for example, comments, replies, issue descriptions, and merge request @@ -51,7 +53,7 @@ descriptions): | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>p</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>p</kbd> | Toggle Markdown preview when editing text in a text field that has **Write** and **Preview** tabs at the top. | | <kbd>Command</kbd> + <kbd>b</kbd> | <kbd>Control</kbd> + <kbd>b</kbd> | Bold the selected text (surround it with `**`). | | <kbd>Command</kbd> + <kbd>i</kbd> | <kbd>Control</kbd> + <kbd>i</kbd> | Italicize the selected text (surround it with `_`). | -| <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>s</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>s</kbd> | Strike through the selected text (surround it with `~~`). | +| <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>x</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>x</kbd> | Strike through the selected text (surround it with `~~`). | | <kbd>Command</kbd> + <kbd>k</kbd> | <kbd>Control</kbd> + <kbd>k</kbd> | Add a link (surround the selected text with `[]()`). | The shortcuts for editing in text fields are always enabled, even if other @@ -79,13 +81,14 @@ relatively quickly to work, and they take you to another page in the project. | <kbd>g</kbd> + <kbd>j</kbd> | Go to the CI/CD jobs list (**CI/CD > Jobs**). | | <kbd>g</kbd> + <kbd>l</kbd> | Go to the project metrics (**Monitor > Metrics**). | | <kbd>g</kbd> + <kbd>e</kbd> | Go to the project environments (**Deployments > Environments**). | -| <kbd>g</kbd> + <kbd>k</kbd> | Go to the project Kubernetes cluster integration page (**Infrastructure > Kubernetes clusters**). Note that you must have at least [`maintainer` permissions](permissions.md) to access this page. | +| <kbd>g</kbd> + <kbd>k</kbd> | Go to the project Kubernetes cluster integration page (**Infrastructure > Kubernetes clusters**). You must have at least [`maintainer` permissions](permissions.md) to access this page. | | <kbd>g</kbd> + <kbd>s</kbd> | Go to the project snippets list (**Snippets**). | | <kbd>g</kbd> + <kbd>w</kbd> | Go to the [project wiki](project/wiki/index.md) (**Wiki**), if enabled. | +| <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | -### Issues and merge requests +### Issues -These shortcuts are available when viewing issues and [merge requests](project/merge_requests/index.md): +These shortcuts are available when viewing issues: | Keyboard shortcut | Description | |------------------------------|-------------| @@ -94,18 +97,26 @@ These shortcuts are available when viewing issues and [merge requests](project/m | <kbd>m</kbd> | Change milestone. | | <kbd>l</kbd> | Change label. | | <kbd>r</kbd> | Start writing a comment. Pre-selected text is quoted in the comment. Can't be used to reply in a thread. | -| <kbd>n</kbd> | Move to next unresolved discussion (merge requests only). | -| <kbd>p</kbd> | Move to previous unresolved discussion (merge requests only). | -| <kbd>]</kbd> or <kbd>j</kbd> | Move to next file (merge requests only). | -| <kbd>[</kbd> or <kbd>k</kbd> | Move to previous file (merge requests only). | -| <kbd>b</kbd> | Copy source branch name (merge requests only). | | <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | - -Merge requests additionally support the following shortcuts: - -| macOS shortcut | Windows shortcut | Description | -|---------------------------------|---------------------|-------------| -| <kbd>Command</kbd> + <kbd>p</kbd> | <kbd>Control</kbd> + <kbd>p</kbd> | Search for, and then jump to a file for review. | +| <kbd>→</kbd> | Go to the next design. | +| <kbd>←</kbd> | Go to the previous design. | +| <kbd>Escape</kbd> | Close the design. | + +### Merge requests + +These shortcuts are available when viewing [merge requests](project/merge_requests/index.md): + +| macOS shortcut | Windows shortcut | Description | +|-----------------------------------|---------------------|-------------| +| <kbd>]</kbd> or <kbd>j</kbd> | | Move to next file. | +| <kbd>[</kbd> or <kbd>k</kbd> | | Move to previous file. | +| <kbd>Command</kbd> + <kbd>p</kbd> | <kbd>Control</kbd> + <kbd>p</kbd> | Search for, and then jump to a file for review. | +| <kbd>n</kbd> | | Move to next unresolved discussion. | +| <kbd>p</kbd> | | Move to previous unresolved discussion. | +| <kbd>b</kbd> | | Copy source branch name. | +| <kbd>r</kbd> | | Start writing a comment. Pre-selected text is quoted in the comment. Can't be used to reply in a thread. | +| <kbd>c</kbd> | | Move to next commit. | +| <kbd>x</kbd> | | Move to previous commit. | ### Project files @@ -119,15 +130,88 @@ These shortcuts are available when browsing the files in a project (go to | <kbd>Enter</kbd> | Open selection. | | <kbd>Escape</kbd> | Go back to file list screen (only while searching for files, **Repository > Files**, then select **Find File**). | | <kbd>y</kbd> | Go to file permalink (only while viewing a file). | -| <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | +| <kbd>.</kbd> | Open the [Web IDE](project/web_ide/index.md). | ### Web IDE These shortcuts are available when editing a file with the [Web IDE](project/web_ide/index.md): -| macOS shortcut | Windows shortcut | Description | +| macOS shortcut | Windows/Linux shortcut | Description | |---------------------------------|---------------------|-------------| -| <kbd>Command</kbd> + <kbd>p</kbd> | <kbd>Control</kbd> + <kbd>p</kbd> | Search for, and then open another file for editing. | +| <kbd>Option</kbd> + <kbd>Command</kbd> + <kbd>↑</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>↑</kbd> | Add cursor above | +| <kbd>Option</kbd> + <kbd>Command</kbd> + <kbd>↓</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>↓</kbd> | Add cursor below | +| <kbd>Shift</kbd> + <kbd>Option</kbd> + <kbd>I</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>I</kbd> | Add cursors to line ends | +| <kbd>Command</kbd> + <kbd>K</kbd>, <kbd>Command</kbd> + <kbd>C</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, <kbd>Control</kbd> + <kbd>C</kbd> _or_ <kbd>Control</kbd> + <kbd>/</kbd> | Add line comment | +| <kbd>Command</kbd> + <kbd>D</kbd> | <kbd>Control</kbd> + <kbd>D</kbd> | Add selection to next find match | +| <kbd>Command</kbd> + <kbd>F2</kbd> | <kbd>Control</kbd> + <kbd>F2</kbd> | Change all occurrences | +| <kbd>F1</kbd> | <kbd>F1</kbd> | Command palette | +| <kbd>Shift</kbd> + <kbd>Option</kbd> + <kbd>↓</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>↓</kbd> | Copy line down | +| <kbd>Shift</kbd> + <kbd>Option</kbd> + <kbd>↑</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>↑</kbd> | Copy line up [(Linux note)](#linux-shortcuts) | +| <kbd>Command</kbd> + <kbd>U</kbd> | <kbd>Control</kbd> + <kbd>U</kbd> | Cursor undo | +| <kbd>Command</kbd> + <kbd>Backspace</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Backspace</kbd> | Delete all left | +| <kbd>Control</kbd> + <kbd>K</kbd> | | Delete all right | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>K</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>K</kbd> | Delete line | +| | <kbd>Control</kbd> + <kbd>Backspace</kbd> | Delete word | +| <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>→</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>→</kbd> | Expand selection | +| <kbd>Command</kbd> + <kbd>P</kbd> | <kbd>Control</kbd> + <kbd>P</kbd> | File finder | +| <kbd>Command</kbd> + <kbd>F</kbd> | <kbd>Control</kbd> + <kbd>F</kbd> | Find | +| <kbd>Enter</kbd> | <kbd>Enter</kbd> or <kbd>F3</kbd> | Find next | +| <kbd>Command</kbd> + <kbd>F3</kbd> | <kbd>F3</kbd> | Find next selection [(Linux note)](#linux-shortcuts) | +| <kbd>Shift</kbd> + <kbd>Enter</kbd> + <kbd>F3</kbd> | <kbd>Shift</kbd> + <kbd>F3</kbd> | Find previous | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>F3</kbd> | <kbd>Shift</kbd> + <kbd>F3</kbd> | Find previous selection | +| <kbd>Command</kbd> + <kbd>E</kbd> | | Find with selection | +| <kbd>Option</kbd> + <kbd>Command</kbd> + <kbd>[</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>[</kbd> | Fold | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>O</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>O</kbd> | Fold all | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>/</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>/</kbd> | Fold all block comments | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>8</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>8</kbd> | Fold all regions | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>-</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>-</kbd> | Fold all regions except selected | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>1</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>1</kbd> | Fold level 1 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>2</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>2</kbd> | Fold level 2 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>3</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>3</kbd> | Fold level 3 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>4</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>4</kbd> | Fold level 4 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>5</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>5</kbd> | Fold level 5 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>6</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>6</kbd> | Fold level 6 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>7</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>7</kbd> | Fold level 7 | +| <kbd>Command</kbd> + <kbd>K</kbd> , then <kbd>Command</kbd> + <kbd>[</kbd> | <kbd>Control</kbd> + <kbd>K</kbd> , then <kbd>Control</kbd> + <kbd>[</kbd> | Fold recursively | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>\</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>\</kbd> | Go to bracket | +| <kbd>Control</kbd> + <kbd>G</kbd> | <kbd>Control</kbd> + <kbd>G</kbd> | Go to line or column | +| <kbd>Option</kbd> + <kbd>F8</kbd> | <kbd>Alt</kbd> + <kbd>F8</kbd> | Go to next problem (error, warning, information) | +| <kbd>F8</kbd> | <kbd>F8</kbd> | Go to next problem in files (error, warning, information) | +| <kbd>Shift</kbd> + <kbd>Option</kbd> + <kbd>F8</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>F8</kbd> | Go to previous problem (error, warning, information) | +| <kbd>Shift</kbd> + <kbd>F8</kbd> | <kbd>Shift</kbd> + <kbd>F8</kbd> | Go to previous problem in files (error, warning, information) | +| <kbd>Command</kbd> + <kbd>]</kbd> | <kbd>Control</kbd> + <kbd>]</kbd> | Indent line | +| <kbd>Shift</kbd> + <kbd>Command</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Enter</kbd> | Insert line above | +| <kbd>Command</kbd> + <kbd>Enter</kbd> | <kbd>Control</kbd> + <kbd>Enter</kbd> | Insert line below | +| <kbd>Control</kbd> + <kbd>J</kbd> | <kbd>Control</kbd> + <kbd>J</kbd> | Join lines [(Linux note)](#linux-shortcuts) | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>D</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>D</kbd> | Move last selection to next find match | +| <kbd>Option</kbd> + <kbd>↓</kbd> | <kbd>Alt</kbd> + <kbd>↓</kbd> | Move line down | +| <kbd>Option</kbd> + <kbd>↑</kbd> | <kbd>Alt</kbd> + <kbd>↑</kbd> | Move line up | +| <kbd>Command</kbd> + <kbd>[</kbd> | <kbd>Control</kbd> + <kbd>[</kbd> | Outdent line | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>P</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>P</kbd> | Preview Markdown [(Linux note)](#linux-shortcuts) | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>U</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>U</kbd> or <kbd>Control</kbd> + <kbd>/</kbd> | Remove line comment | +| <kbd>Option</kbd> + <kbd>Command</kbd> + <kbd>F</kbd> | <kbd>Control</kbd> + <kbd>F</kbd> | Replace | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>.</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>.</kbd> | Replace with next value | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>,</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>,</kbd> | Replace with previous value | +| <kbd>Command</kbd> + <kbd>S</kbd> | <kbd>Control</kbd> + <kbd>S</kbd> | Save files | +| <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>L</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>L</kbd> | Select all occurrences of find match | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>B</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>B</kbd> | Set selection anchor | +| <kbd>Option</kbd> + <kbd>F1</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>F1</kbd> | Show accessibility help | +| <kbd>Shift</kbd> + <kbd>F10</kbd> | <kbd>Shift</kbd> + <kbd>F10</kbd> | Show editor context menu | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>I</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>I</kbd> | Show hover | +| <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>Command</kbd> + <kbd>←</kbd> | <kbd>Shift</kbd> + <kbd>Alt</kbd> + <kbd>←</kbd> | Shrink selection | +| <kbd>Shift</kbd> + <kbd>Option</kbd> + <kbd>A</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>A</kbd> | Toggle block comment | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>L</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>L</kbd> | Toggle fold | +| <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>M</kbd> | <kbd>Control</kbd> + <kbd>M</kbd> | Toggle Tab key moves focus | +| <kbd>Command</kbd> + <kbd>/</kbd> | <kbd>Control</kbd> + <kbd>/</kbd> | Toggle line comment | +| <kbd>Control</kbd> + <kbd>T</kbd> | | Transpose letters | +| <kbd>Control</kbd> + <kbd>Space</kbd> | <kbd>Control</kbd> + <kbd>Space</kbd> | Trigger Suggest | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>X</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>X</kbd> | Trim trailing whitespace | +| <kbd>Option</kbd> + <kbd>Command</kbd> + <kbd>]</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>]</kbd> | Unfold | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>J</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>J</kbd> | Unfold all | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>9</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>9</kbd> | Unfold all regions | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>=</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>=</kbd> | Unfold all regions except selected | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>]</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>]</kbd> | Unfold recursively | +| <kbd>Command</kbd> + <kbd>K</kbd>, then <kbd>Command</kbd> + <kbd>X</kbd> | <kbd>Control</kbd> + <kbd>K</kbd>, then <kbd>Control</kbd> + <kbd>X</kbd> | Trim trailing whitespace | | <kbd>Command</kbd> + <kbd>Enter</kbd> | <kbd>Control</kbd> + <kbd>Enter</kbd> | Commit (when editing the commit message). | ### Repository graph @@ -148,8 +232,8 @@ page (go to **Repository > Graph**): This shortcut is available when viewing a [wiki page](project/wiki/index.md): -| Keyboard shortcut | Description | -|-------------------|-------------| +| Keyboard shortcut | Description | +|-------------------|-----------------| | <kbd>e</kbd> | Edit wiki page. | ### Content editor @@ -183,7 +267,7 @@ These shortcuts are available when editing a file with the | <kbd>Command</kbd> + <kbd>Alt</kbd> + <kbd>5</kbd> | <kbd>Control</kbd> + <kbd>Alt</kbd> + <kbd>5</kbd> | Apply heading style 5 | | <kbd>Command</kbd> + <kbd>Alt</kbd> + <kbd>6</kbd> | <kbd>Control</kbd> + <kbd>Alt</kbd> + <kbd>6</kbd> | Apply heading style 6 | | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>7</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>7</kbd> | Ordered list | -| <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>8</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>8</kbd> | Bullet list | +| <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>8</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>8</kbd> | Unordered list | | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>9</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>9</kbd> | Task list | | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>b</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>b</kbd> | Blockquote | | <kbd>Command</kbd> + <kbd>Alt</kbd> + <kbd>c</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>c</kbd> | Code block | @@ -194,13 +278,13 @@ These shortcuts are available when editing a file with the #### Text selection -| macOS shortcut | Windows shortcut | Description | -|----------------|------------------|-------------| -| <kbd>Command</kbd> + <kbd>a</kbd> | <kbd>Control</kbd> + <kbd>a</kbd> | Select all | -| <kbd>Shift</kbd> + <kbd>←</kbd> | <kbd>Shift</kbd> + <kbd>←</kbd> | Extend selection one character to left | -| <kbd>Shift</kbd> + <kbd>→</kbd> | <kbd>Shift</kbd> + <kbd>→</kbd> | Extend selection one character to right | -| <kbd>Shift</kbd> + <kbd>↑</kbd> | <kbd>Shift</kbd> + <kbd>↑</kbd> | Extend selection one line up | -| <kbd>Shift</kbd> + <kbd>↓</kbd> | <kbd>Shift</kbd> + <kbd>↓</kbd> | Extend selection one line down | +| macOS shortcut | Windows shortcut | Description | +|-----------------------------------|-----------------------------------|-------------| +| <kbd>Command</kbd> + <kbd>a</kbd> | <kbd>Control</kbd> + <kbd>a</kbd> | Select all | +| <kbd>Shift</kbd> + <kbd>←</kbd> | <kbd>Shift</kbd> + <kbd>←</kbd> | Extend selection one character to left | +| <kbd>Shift</kbd> + <kbd>→</kbd> | <kbd>Shift</kbd> + <kbd>→</kbd> | Extend selection one character to right | +| <kbd>Shift</kbd> + <kbd>↑</kbd> | <kbd>Shift</kbd> + <kbd>↑</kbd> | Extend selection one line up | +| <kbd>Shift</kbd> + <kbd>↓</kbd> | <kbd>Shift</kbd> + <kbd>↓</kbd> | Extend selection one line down | | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>↑</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>↑</kbd> | Extend selection to the beginning of the document | | <kbd>Command</kbd> + <kbd>Shift</kbd> + <kbd>↓</kbd> | <kbd>Control</kbd> + <kbd>Shift</kbd> + <kbd>↓</kbd> | Extend selection to the end of the document | @@ -217,11 +301,23 @@ These shortcuts are available when using a [filtered search input](search/index. These shortcuts are available when viewing [epics](group/epics/index.md): -| Keyboard shortcut | Description | -|-------------------|-------------| +| Keyboard shortcut | Description | +|-------------------|-------------------| | <kbd>r</kbd> | Start writing a comment. Pre-selected text is quoted in the comment. Can't be used to reply in a thread. | | <kbd>e</kbd> | Edit description. | -| <kbd>l</kbd> | Change label. | +| <kbd>l</kbd> | Change label. | + +## Metrics + +These shortcuts are available when using metrics: + +| Keyboard shortcut | Description | +|-------------------|---------------------| +| <kbd>e</kbd> | Expand panel. | +| <kbd>l</kbd> | View logs. | +| <kbd>d</kbd> | Download CSV. | +| <kbd>c</kbd> | Copy link to chart. | +| <kbd>a</kbd> | Alerts. | ## Disable keyboard shortcuts @@ -232,3 +328,10 @@ To disable keyboard shortcuts: 1. While viewing a page that supports keyboard shortcuts, and outside a text box, press <kbd>?</kbd> to display the list of shortcuts. 1. Select **Toggle shortcuts**. + +## Troubleshooting + +### Linux shortcuts + +Linux users may encounter GitLab keyboard shortcuts that are overridden by +their operating system, or their browser. diff --git a/doc/user/ssh.md b/doc/user/ssh.md index 54d4722ee2b..27bb7124afe 100644 --- a/doc/user/ssh.md +++ b/doc/user/ssh.md @@ -1,8 +1,7 @@ --- stage: Manage group: Authentication and Authorization -info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" -type: howto, reference +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Use SSH keys to communicate with GitLab **(FREE)** @@ -41,7 +40,7 @@ Administrators can [restrict which keys are permitted and their minimum lengths] The book [Practical Cryptography With Go](https://leanpub.com/gocrypto/read#leanpub-auto-chapter-5-digital-signatures) suggests that [ED25519](https://ed25519.cr.yp.to/) keys are more secure and performant than RSA keys. -OpenSSH 6.5 introduced ED25519 SSH keys in 2014 and they should be available on most +OpenSSH 6.5 introduced ED25519 SSH keys in 2014, and they should be available on most operating systems. ### ED25519_SK SSH keys @@ -60,7 +59,7 @@ must have [OpenSSH 8.2](https://www.openssh.com/releasenotes.html#8.2) or later ### RSA SSH keys -Available documentation suggests that ED25519 is more secure than RSA. +Available documentation suggests ED25519 is more secure than RSA. If you use an RSA key, the US National Institute of Science and Technology in [Publication 800-57 Part 3 (PDF)](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-57Pt3r1.pdf) @@ -71,27 +70,27 @@ Review the `man` page for your installed `ssh-keygen` command for details. Before you create a key pair, see if a key pair already exists. -1. On Windows, Linux, or macOS, go to your home directory. +1. Go to your home directory. 1. Go to the `.ssh/` subdirectory. If the `.ssh/` subdirectory doesn't exist, you are either not in the home directory, or you haven't used `ssh` before. In the latter case, you need to [generate an SSH key pair](#generate-an-ssh-key-pair). 1. See if a file with one of the following formats exists: - | Algorithm | Public key | Private key | - | --------- | ---------- | ----------- | + | Algorithm | Public key | Private key | + |-----------------------|------------|-------------| | ED25519 (preferred) | `id_ed25519.pub` | `id_ed25519` | - | ED25519_SK | `id_ed25519_sk.pub` | `id_ed25519_sk` | - | ECDSA_SK | `id_ecdsa_sk.pub` | `id_ecdsa_sk` | - | RSA (at least 2048-bit key size) | `id_rsa.pub` | `id_rsa` | - | DSA (deprecated) | `id_dsa.pub` | `id_dsa` | - | ECDSA | `id_ecdsa.pub` | `id_ecdsa` | + | ED25519_SK | `id_ed25519_sk.pub` | `id_ed25519_sk` | + | ECDSA_SK | `id_ecdsa_sk.pub` | `id_ecdsa_sk` | + | RSA (at least 2048-bit key size) | `id_rsa.pub` | `id_rsa` | + | DSA (deprecated) | `id_dsa.pub` | `id_dsa` | + | ECDSA | `id_ecdsa.pub` | `id_ecdsa` | ## Generate an SSH key pair -If you do not have an existing SSH key pair, generate a new one. +If you do not have an existing SSH key pair, generate a new one: 1. Open a terminal. -1. Type `ssh-keygen -t` followed by the key type and an optional comment. +1. Run `ssh-keygen -t` followed by the key type and an optional comment. This comment is included in the `.pub` file that's created. You may want to use an email address for the comment. @@ -107,7 +106,7 @@ If you do not have an existing SSH key pair, generate a new one. ssh-keygen -t rsa -b 2048 -C "<comment>" ``` -1. Press Enter. Output similar to the following is displayed: +1. Press <kbd>Enter</kbd>. Output similar to the following is displayed: ```plaintext Generating public/private ed25519 key pair. @@ -126,11 +125,10 @@ If you do not have an existing SSH key pair, generate a new one. Enter same passphrase again: ``` -1. A confirmation is displayed, including information about where your files are stored. + A confirmation is displayed, including information about where your files are stored. -A public and private key are generated. -[Add the public SSH key to your GitLab account](#add-an-ssh-key-to-your-gitlab-account) and keep -the private key secure. +A public and private key are generated. [Add the public SSH key to your GitLab account](#add-an-ssh-key-to-your-gitlab-account) +and keep the private key secure. ### Configure SSH to point to a different directory @@ -158,7 +156,7 @@ configure your SSH client to point to the directory where the private key is sto IdentityFile ~/.ssh/example_com_rsa ``` - For more information on these settings, see the [`man ssh_config`](https://man.openbsd.org/ssh_config) page in the SSH configuration manual. +For more information on these settings, see the [`man ssh_config`](https://man.openbsd.org/ssh_config) page in the SSH configuration manual. Public SSH keys must be unique to GitLab because they bind to your account. Your SSH key is the only identifier you have when you push code with SSH. @@ -166,7 +164,7 @@ It must uniquely map to a single user. ### Update your SSH key passphrase -You can update the passphrase for your SSH key. +You can update the passphrase for your SSH key: 1. Open a terminal and run this command: @@ -174,34 +172,32 @@ You can update the passphrase for your SSH key. ssh-keygen -p -f /path/to/ssh_key ``` -1. At the prompts, type the passphrase and press Enter. +1. At the prompts, enter the passphrase and then press <kbd>Enter</kbd>. ### Upgrade your RSA key pair to a more secure format -If your version of OpenSSH is between 6.5 and 7.8, -you can save your private RSA SSH keys in a more secure -OpenSSH format. - -1. Open a terminal and run this command: +If your version of OpenSSH is between 6.5 and 7.8, you can save your private +RSA SSH keys in a more secure OpenSSH format by opening a terminal and running +this command: - ```shell - ssh-keygen -o -f ~/.ssh/id_rsa - ``` +```shell +ssh-keygen -o -f ~/.ssh/id_rsa +``` - Alternatively, you can generate a new RSA key with the more secure encryption format with - the following command: +Alternatively, you can generate a new RSA key with the more secure encryption format with +the following command: - ```shell - ssh-keygen -o -t rsa -b 4096 -C "<comment>" - ``` +```shell +ssh-keygen -o -t rsa -b 4096 -C "<comment>" +``` ## Generate an SSH key pair for a FIDO/U2F hardware security key -To generate ED25519_SK or ECDSA_SK SSH keys, you must use OpenSSH 8.2 or later. +To generate ED25519_SK or ECDSA_SK SSH keys, you must use OpenSSH 8.2 or later: 1. Insert a hardware security key into your computer. 1. Open a terminal. -1. Type `ssh-keygen -t` followed by the key type and an optional comment. +1. Run `ssh-keygen -t` followed by the key type and an optional comment. This comment is included in the `.pub` file that's created. You may want to use an email address for the comment. @@ -229,7 +225,7 @@ To generate ED25519_SK or ECDSA_SK SSH keys, you must use OpenSSH 8.2 or later. from the security key by [`ssh-add -K`](https://man.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man1/ssh-add.1#K) or [`ssh-keygen -K`](https://man.openbsd.org/cgi-bin/man.cgi/OpenBSD-current/man1/ssh-keygen#K). -1. Select Enter. Output similar to the following is displayed: +1. Press <kbd>Enter</kbd>. Output similar to the following is displayed: ```plaintext Generating public/private ed25519-sk key pair. @@ -251,31 +247,31 @@ To generate ED25519_SK or ECDSA_SK SSH keys, you must use OpenSSH 8.2 or later. Enter same passphrase again: ``` -1. A confirmation is displayed, including information about where your files are stored. + A confirmation is displayed, including information about where your files are stored. A public and private key are generated. [Add the public SSH key to your GitLab account](#add-an-ssh-key-to-your-gitlab-account). ## Add an SSH key to your GitLab account -To use SSH with GitLab, copy your public key to your GitLab account. +To use SSH with GitLab, copy your public key to your GitLab account: 1. Copy the contents of your public key file. You can do this manually or use a script. For example, to copy an ED25519 key to the clipboard: - **macOS:** + **macOS** ```shell tr -d '\n' < ~/.ssh/id_ed25519.pub | pbcopy ``` - **Linux** (requires the `xclip` package): + **Linux** (requires the `xclip` package) ```shell xclip -sel clip < ~/.ssh/id_ed25519.pub ``` - **Git Bash on Windows:** + **Git Bash on Windows** ```shell cat ~/.ssh/id_ed25519.pub | clip @@ -298,8 +294,6 @@ To use SSH with GitLab, copy your public key to your GitLab account. - GitLab 13.12 and earlier, the expiration date is informational only. It doesn't prevent you from using the key. Administrators can view expiration dates and use them for guidance when [deleting keys](admin_area/credentials_inventory.md#delete-a-users-ssh-key). - - GitLab 14.0 and later, the expiration date is enforced. Administrators can - [allow expired keys to be used](admin_area/settings/account_and_limit_settings.md#allow-expired-ssh-keys-to-be-used-deprecated). - GitLab checks all SSH keys at 02:00 AM UTC every day. It emails an expiration notice for all SSH keys that expire on the current date. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322637) in GitLab 13.11.) - GitLab checks all SSH keys at 01:00 AM UTC every day. It emails an expiration notice for all SSH keys that are scheduled to expire seven days from now. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322637) in GitLab 13.11.) 1. Select **Add key**. @@ -308,9 +302,12 @@ To use SSH with GitLab, copy your public key to your GitLab account. Verify that your SSH key was added correctly. +The following commands use the example hostname `gitlab.example.com`. Replace this example hostname with your GitLab instance's hostname, for example, `git@gitlab.com`. + 1. For GitLab.com, to ensure you're connecting to the correct server, confirm the [SSH host keys fingerprints](gitlab_com/index.md#ssh-host-keys-fingerprints). -1. Open a terminal and run this command, replacing `gitlab.example.com` with your GitLab instance URL: +1. Open a terminal and run this command, replacing `gitlab.example.com` with your + GitLab instance URL: ```shell ssh -T git@gitlab.example.com @@ -326,7 +323,7 @@ Verify that your SSH key was added correctly. Warning: Permanently added 'gitlab.example.com' (ECDSA) to the list of known hosts. ``` - Type `yes` and press Enter. + Type `yes` and press <kbd>Enter</kbd>. 1. Run the `ssh -T git@gitlab.example.com` command again. You should receive a _Welcome to GitLab, `@username`!_ message. @@ -352,10 +349,10 @@ on `ssh` command options, see the `man` pages for both `ssh` and `ssh_config`. ## Use different accounts on a single GitLab instance -You can use multiple accounts to connect to a single instance of GitLab. -You can do this by using the command in the [previous topic](#use-different-keys-for-different-repositories). -However, even if you set `IdentitiesOnly` to `yes`, you cannot sign in if an `IdentityFile` exists -outside of a `Host` block. +You can use multiple accounts to connect to a single instance of GitLab. You +can do this by using the command in the [previous topic](#use-different-keys-for-different-repositories). +However, even if you set `IdentitiesOnly` to `yes`, you cannot sign in if an +`IdentityFile` exists outside of a `Host` block. Instead, you can assign aliases to hosts in the `~.ssh/config` file. diff --git a/doc/user/todos.md b/doc/user/todos.md index 5cea619c830..c261d719da0 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -84,28 +84,25 @@ You can manually add an item to your To-Do List.  -## Create a to-do item by directly addressing someone +## Create a to-do item by mentioning someone -You can create a to-do item by directly addressing someone at the start of a line. -For example, in the following comment: +You can create a to-do item by mentioning someone anywhere except for a code block. Mentioning a user many times in one message only creates one to-do item. -```markdown +For example, from the following comment, everyone except `frank` gets a to-do item created for them: + +````markdown @alice What do you think? cc: @bob - @carol can you please have a look? > @dan what do you think? -@erin @frank thank you! -``` - -The people who receive to-do items are `@alice`, `@erin`, and -`@frank`. +Hey @erin, this is what they said: -To view to-do items where a user was directly addressed, go to the To-Do List and -from the **Action** filter, select **Directly addressed**. - -Mentioning a user many times only creates one to-do item. +``` +Hi, please message @frank :incoming_envelope: +``` +```` ## Actions that mark a to-do item as done diff --git a/doc/user/usage_quotas.md b/doc/user/usage_quotas.md index 84a2449f481..21aa93d3f8b 100644 --- a/doc/user/usage_quotas.md +++ b/doc/user/usage_quotas.md @@ -48,6 +48,19 @@ The following storage usage statistics are available to a maintainer: - Total excess storage used: Total amount of storage used that exceeds their allocated storage. - Purchased storage available: Total storage that has been purchased but is not yet used. +## Manage your storage usage + +You can use several methods to manage and reduce your usage for some storage types. + +For more information, see the following pages: + +- [Reduce package registry storage](packages/package_registry/reduce_package_registry_storage.md) +- [Reduce dependency proxy storage](packages/dependency_proxy/reduce_dependency_proxy_storage.md) +- [Reduce repository size](project/repository/reducing_the_repo_size_using_git.md) +- [Reduce container registry storage](packages/container_registry/reduce_container_registry_storage.md) +- [Reduce container registry data transfers](packages/container_registry/reduce_container_registry_data_transfer.md) +- [Reduce wiki repository size](../administration/wikis/index.md#reduce-wiki-repository-size) + ## Excess storage usage Excess storage usage is the amount that a project's repository exceeds the free storage quota. If no |
