diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 11:10:13 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 11:10:13 +0000 |
| commit | 0ea3fcec397b69815975647f5e2aa5fe944a8486 (patch) | |
| tree | 7979381b89d26011bcf9bdc989a40fcc2f1ed4ff /doc | |
| parent | 72123183a20411a36d607d70b12d57c484394c8e (diff) | |
| download | gitlab-ce-0ea3fcec397b69815975647f5e2aa5fe944a8486.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42
Diffstat (limited to 'doc')
835 files changed, 15203 insertions, 9322 deletions
diff --git a/doc/.vale/gitlab/BadPlurals.yml b/doc/.vale/gitlab/BadPlurals.yml index efc55ac3d56..63f002fec94 100644 --- a/doc/.vale/gitlab/BadPlurals.yml +++ b/doc/.vale/gitlab/BadPlurals.yml @@ -6,9 +6,8 @@ # For a list of all options, see https://docs.errata.ai/vale/styles extends: existence message: 'Rewrite "%s" to be plural, without parentheses.' -link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html#s level: warning -scope: raw ignorecase: true raw: - '\b\w+\(s\)(?<!http\(s\))' diff --git a/doc/.vale/gitlab/EOLWhitespace.yml b/doc/.vale/gitlab/EOLWhitespace.yml new file mode 100644 index 00000000000..483db0cafe6 --- /dev/null +++ b/doc/.vale/gitlab/EOLWhitespace.yml @@ -0,0 +1,13 @@ +--- +# Error: gitlab.EOLWhitespace +# +# Checks that there is no useless whitespace at the end of lines. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Lines should not end with whitespace characters.' +link: https://docs.gitlab.com/ee/development/documentation/versions.html +level: warning +scope: raw +raw: + - ' +\n' diff --git a/doc/.vale/gitlab/OutdatedVersions.yml b/doc/.vale/gitlab/OutdatedVersions.yml index 15ae0a5814a..c78c045a1fd 100644 --- a/doc/.vale/gitlab/OutdatedVersions.yml +++ b/doc/.vale/gitlab/OutdatedVersions.yml @@ -21,3 +21,4 @@ tokens: - "GitLab (v)?9." - "GitLab (v)?10." - "GitLab (v)?11." + - "GitLab (v)?12." diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index a0670d75f18..af426fa7e06 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -38,7 +38,7 @@ swap: to sign-in: to sign in can sign-in: can sign in x509: X.509 - yaml: YAML + yml: YAML admin user: administrator admin users: administrators administrator permission: administrator access diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index 6093565fd7c..bdbee8108d5 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -6,7 +6,7 @@ # For a list of all options, see https://docs.errata.ai/vale/styles. extends: conditional message: "'%s' is uppercase. Use lowercase or `backticks` if possible. Otherwise add this word to the rule's exception list." -link: https://about.gitlab.com/handbook/marketing/growth-marketing/content/editorial-team/#acronyms +link: https://docs.gitlab.com/ee/development/documentation/testing.html#vale-uppercase-acronym-test level: warning ignorecase: false # Ensures that the existence of 'first' implies the existence of 'second'. @@ -14,8 +14,10 @@ first: '\b([A-Z]{3,5})\b' second: '(?:\b[A-Z][a-z]+ )+\(([A-Z]{3,5})\)' # ... with the exception of these: exceptions: + - ACL - AJAX - ANSI + - APAC - API - APM - ARM @@ -26,26 +28,36 @@ exceptions: - BSD - CAS - CDN + - CGI - CIDR - CLI - CNA - - CNAME + - CNCF - CORE + - CORS - CPU - CRIME + - CRM + - CRUD - CSRF - CSS - CSV + - CTE + - CWE - CVE - CVS + - CVSS - DAG - DAST + - DDL - DHCP + - DML - DNS - DOM - DSA - DSL - DVCS + - DVD - ECDSA - ECS - EFS @@ -70,10 +82,12 @@ exceptions: - GID - GIF - GKE + - GLEX - GLFM - GNU - GPG - GPL + - GPS - GPU - GUI - HAML @@ -81,6 +95,7 @@ exceptions: - HEAD - HIPAA - HLL + - HSTS - HTML - HTTP - HTTPS @@ -90,6 +105,7 @@ exceptions: - ICO - IDE - IID + - IIS - IMAP - IOPS - IRC @@ -99,16 +115,19 @@ exceptions: - JSON - JVM - JWT + - KICS - LAN - LDAP - LDAPS - LESS - LFS - LRU + - LSIF - LTM - LTS - MIME - MIT + - MITRE - MVC - NAT - NDA @@ -117,6 +136,7 @@ exceptions: - NOTE - NPM - NTP + - OKD - ONLY - OSS - OTP @@ -133,7 +153,9 @@ exceptions: - PNG - POSIX - POST + - PROXY - PUT + - QPS - RAID - RAM - RBAC @@ -155,7 +177,9 @@ exceptions: - SAN - SAST - SATA + - SBOM - SCIM + - SCM - SCP - SCSS - SDK @@ -163,13 +187,17 @@ exceptions: - SEO - SFTP - SHA + - SKI - SLA - SLI + - SLO - SMS - SMTP + - SOAP - SOC - SOX - SPDX + - SPDY - SPF - SQL - SRE @@ -179,6 +207,7 @@ exceptions: - SSL - SSO - STI + - SUSE - SVG - SVN - TCP @@ -189,7 +218,9 @@ exceptions: - TODO - TOML - TOTP + - TPS - TTL + - UBI - UDP - UID - UID diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 92e10a91753..de7fb45490d 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -73,6 +73,7 @@ blockquoting boolean booleans Bootsnap +Bottlerocket browsable bugfix bugfixed @@ -571,7 +572,7 @@ rsyncing rsyncs Rubinius Rubix -Rubocop +RuboCop Rubular ruleset rulesets diff --git a/doc/administration/application_settings_cache.md b/doc/administration/application_settings_cache.md index b3aac932fa8..6c58e6886c4 100644 --- a/doc/administration/application_settings_cache.md +++ b/doc/administration/application_settings_cache.md @@ -1,10 +1,10 @@ --- -stage: Enablement +stage: Data Stores group: Memory info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Change the expiration interval for application cache **(FREE SELF)** +# Application cache interval **(FREE SELF)** By default, GitLab caches application settings for 60 seconds. Occasionally, you may need to increase that interval to have more delay between application @@ -14,6 +14,8 @@ We recommend you set this value to greater than `0` seconds. Setting it to `0` causes the `application_settings` table to load for every request. This causes extra load for Redis and PostgreSQL. +## Change the expiration interval for application cache + To change the expiry value: **For Omnibus installations** @@ -32,8 +34,6 @@ To change the expiry value: gitlab-ctl restart ``` ---- - **For installations from source** 1. Edit `config/gitlab.yml`: diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 4678e23dfc9..ad235ead992 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -10,11 +10,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338939) in GitLab 14.7. > - [Feature flag `ff_external_audit_events_namespace`](https://gitlab.com/gitlab-org/gitlab/-/issues/349588) removed in GitLab 14.8. -Event streaming allows owners of top-level groups to set an HTTP endpoint to receive **all** audit events about the group, and its -subgroups and projects as structured JSON. +Users can set an HTTP endpoint for a top-level group to receive all audit events about the group, its subgroups, and +projects as structured JSON. -Top-level group owners can manage their audit logs in third-party systems such as Splunk, using the Splunk -[HTTP Event Collector](https://docs.splunk.com/Documentation/Splunk/8.2.2/Data/UsetheHTTPEventCollector). Any service that can receive +Top-level group owners can manage their audit logs in third-party systems. Any service that can receive structured JSON data can be used as the endpoint. NOTE: @@ -132,7 +131,7 @@ Delete an event streaming destination by specifying an ID. Get the required ID b streaming destinations. ```graphql -mutation { +mutation { externalAuditEventDestinationDestroy(input: { id: destination }) { errors } @@ -144,6 +143,43 @@ Destination is deleted if: - The returned `errors` object is empty. - The API responds with `200 OK`. +## Custom HTTP header values + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361216) in GitLab 15.1 [with a flag](feature_flags.md) named `streaming_audit_event_headers`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `streaming_audit_event_headers`. +On GitLab.com, this feature is not available. +The feature is not ready for production use. + +Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. + +### Add with the API + +Group owners can add a HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. + +```graphql +mutation { + auditEventsStreamingHeadersCreate(input: { destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601", key: "foo", value: "bar" }) { + errors + } +} +``` + +### Delete with the API + +Group owners can remove a HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. + +```graphql +mutation { + auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601" }) { + errors + } +} +``` + +The header is created if the returned `errors` object is empty. + ## Verify event authenticity > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345424) in GitLab 14.8. @@ -158,9 +194,11 @@ the destination's value when [listing streaming destinations](#list-streaming-de > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.0. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.1 by default. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. On GitLab.com, this feature is available. +On self-managed GitLab, by default this feature is available. To hide the +feature, ask an administrator to [disable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. Streaming audit events can be sent when signed-in users push or pull a project's remote Git repositories: diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 6fac21ef889..a329adbed22 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -115,8 +115,14 @@ From there, you can see the following actions: - Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. - Group deploy token was successfully created, revoked, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. - Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- [IP restrictions](../user/group/index.md#restrict-group-access-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0 +- [IP restrictions](../user/group/index.md#group-access-restriction-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. - Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge request approvals settings: + - Prevent approval by author. + - Prevent approvals by users who add commits. + - Prevent editing approval rules in projects and merge requests. + - Require user password to approve. + - Remove all approvals when commits are added to the source branch. Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events) @@ -228,6 +234,7 @@ The following user actions are recorded: - Administrator added or removed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323905) in GitLab 14.1) - Removed SSH key ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1) - Added or removed GPG key ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1) +- A user's two-factor authentication was disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238177) in GitLab 15.1) Instance events can also be accessed via the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index 1d0aff51a04..d82683e1778 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -14,7 +14,7 @@ Users with auditor access have read-only access to all groups, projects, and oth For more information, see [Auditor user permissions and restrictions](#auditor-user-permissions-and-restrictions) section. -Situations where auditor access for users could be helpful include: +Situations where auditor access for users could be helpful include: - Your compliance department wants to run tests against the entire GitLab base to ensure users are complying with password, credit card, and other sensitive diff --git a/doc/administration/auth/authentiq.md b/doc/administration/auth/authentiq.md index a4a085a494c..61ce2b5ab25 100644 --- a/doc/administration/auth/authentiq.md +++ b/doc/administration/auth/authentiq.md @@ -69,7 +69,7 @@ Authentiq generates a Client ID and the accompanying Client Secret for you to us 1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. -On the sign in page there should now be an Authentiq icon below the regular sign in form. Click the +On the sign in page there should now be an Authentiq icon below the regular sign in form. Select the icon to begin the authentication process. If the user: - Has the Authentiq ID app installed in their iOS or Android device, they can: diff --git a/doc/administration/auth/cognito.md b/doc/administration/auth/cognito.md index 66f8407894c..db8cdd3e7bb 100644 --- a/doc/administration/auth/cognito.md +++ b/doc/administration/auth/cognito.md @@ -18,13 +18,13 @@ The following steps enable AWS Cognito as an authentication provider: 1. Sign in to the [AWS console](https://console.aws.amazon.com/console/home). 1. Select **Cognito** from the **Services** menu. -1. Select **Manage User Pools**, and click the **Create a user pool** button in the top right corner. -1. Enter the pool name and then click the **Step through settings** button. +1. Select **Manage User Pools**, and select the **Create a user pool** button in the top right corner. +1. Enter the pool name and then select the **Step through settings** button. 1. Under **How do you want your end users to sign in?**, select **Email address or phone number** and **Allow email addresses**. 1. Under **Which standard attributes do you want to require?**, select **email**. 1. Go to the next steps of configuration and set the rest of the settings to suit your needs - in the basic setup they are not related to GitLab configuration. -1. In the **App clients** settings, click **Add an app client**, add **App client name** and select the **Enable username password based authentication** checkbox. -1. Click **Create app client**. +1. In the **App clients** settings, select **Add an app client**, add **App client name** and select the **Enable username password based authentication** checkbox. +1. Select **Create app client**. 1. In the next step, you can set up AWS Lambda functions for sending emails. You can then finish creating the pool. 1. After creating the user pool, go to **App client settings** and provide the required information: @@ -85,7 +85,7 @@ Include the code block in the `/etc/gitlab/gitlab.rb` file: 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. Your sign-in page should now display a Cognito button below the regular sign-in form. -To begin the authentication process, click the icon, and AWS Cognito asks the user to sign in and authorize the GitLab application. +To begin the authentication process, select the icon, and AWS Cognito asks the user to sign in and authorize the GitLab application. If successful, the user is redirected and signed in to your GitLab instance. For more information, see [Configure initial settings](../../integration/omniauth.md#configure-initial-settings). diff --git a/doc/administration/auth/index.md b/doc/administration/auth/index.md index fca4f163075..ad8d78fdfd6 100644 --- a/doc/administration/auth/index.md +++ b/doc/administration/auth/index.md @@ -8,31 +8,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab authentication and authorization **(FREE SELF)** -GitLab integrates with the following external authentication and authorization -providers: - -- [Atlassian](atlassian.md) -- [Auth0](../../integration/auth0.md) -- [Authentiq](authentiq.md) -- [AWS Cognito](cognito.md) -- [Azure](../../integration/azure.md) -- [Bitbucket Cloud](../../integration/bitbucket.md) -- [CAS](../../integration/cas.md) -- [Crowd](crowd.md) -- [Facebook](../../integration/facebook.md) -- [GitHub](../../integration/github.md) -- [GitLab.com](../../integration/gitlab.md) -- [Google OAuth](../../integration/google.md) -- [JWT](jwt.md) -- [Kerberos](../../integration/kerberos.md) +GitLab integrates with a number of [OmniAuth providers](../../integration/omniauth.md#supported-providers), +and the following external authentication and authorization providers: + - [LDAP](ldap/index.md): Includes Active Directory, Apple Open Directory, Open LDAP, and 389 Server. - [Google Secure LDAP](ldap/google_secure_ldap.md) -- [Salesforce](../../integration/salesforce.md) -- [SAML](../../integration/saml.md) - [SAML for GitLab.com groups](../../user/group/saml_sso/index.md) **(PREMIUM SAAS)** - [Smartcard](smartcard.md) **(PREMIUM SELF)** -- [Twitter](../../integration/twitter.md) NOTE: UltraAuth has removed their software which supports OmniAuth integration. We have therefore removed all references to UltraAuth integration. @@ -47,7 +30,7 @@ For more information, see the links shown on this page for each external provide | **User Provisioning** | SCIM<br>Just-In-Time (JIT) Provisioning | LDAP Sync | | **User Detail Updating** (not group management) | Not Available | LDAP Sync | | **Authentication** | SAML at top-level group (1 provider) | LDAP (multiple providers)<br>Generic OAuth2<br>SAML (only 1 permitted per unique provider)<br>Kerberos<br>JWT<br>Smartcard<br>OmniAuth Providers (only 1 permitted per unique provider) | -| **Provider-to-GitLab Role Sync** | SAML Group Sync | LDAP Group Sync | +| **Provider-to-GitLab Role Sync** | SAML Group Sync | LDAP Group Sync<br>SAML Group Sync ([GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/285150) and later) | | **User Removal** | SCIM (remove user from top-level group) | LDAP (Blocking User from Instance) | ## Change apps or configuration diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 98f928fd7ee..99cba3f220d 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -74,7 +74,7 @@ JWT provides you with a secret key for you to use. installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a JWT icon below the regular sign in form. -Click the icon to begin the authentication process. JWT asks the user to +Select the icon to begin the authentication process. JWT asks the user to sign in and authorize the GitLab application. If everything goes well, the user is redirected to GitLab and signed in. diff --git a/doc/administration/auth/ldap/google_secure_ldap.md b/doc/administration/auth/ldap/google_secure_ldap.md index 10745c5e2bf..7f399f7e730 100644 --- a/doc/administration/auth/ldap/google_secure_ldap.md +++ b/doc/administration/auth/ldap/google_secure_ldap.md @@ -26,7 +26,7 @@ The steps below cover: 1. Provide an `LDAP client name` and an optional `Description`. Any descriptive values are acceptable. For example, the name could be 'GitLab' and the - description could be 'GitLab LDAP Client'. Click the **Continue** button. + description could be 'GitLab LDAP Client'. Select the **Continue** button.  @@ -42,15 +42,15 @@ The steps below cover: 1. Download the generated certificate. This is required for GitLab to communicate with the Google Secure LDAP service. Save the downloaded certificates - for later use. After downloading, click the **Continue to Client Details** button. + for later use. After downloading, select the **Continue to Client Details** button. 1. Expand the **Service Status** section and turn the LDAP client 'ON for everyone'. - After selecting 'Save', click on the 'Service Status' bar again to collapse + After selecting 'Save', select the 'Service Status' bar again to collapse and return to the rest of the settings. 1. Expand the **Authentication** section and choose 'Generate New Credentials'. - Copy/note these credentials for later use. After selecting 'Close', click - on the 'Authentication' bar again to collapse and return to the rest of the settings. + Copy/note these credentials for later use. After selecting 'Close', select + the 'Authentication' bar again to collapse and return to the rest of the settings. Now the Google Secure LDAP Client configuration is finished. The screenshot below shows an example of the final settings. Continue on to configure GitLab. diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 9232e79b800..55b57193bf3 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -169,7 +169,7 @@ These configuration settings are available: | `host` | IP address or domain name of your LDAP server. Ignored when `hosts` is defined. | **{check-circle}** Yes | `'ldap.mydomain.com'` | | `port` | The port to connect with on your LDAP server. Always an integer, not a string. Ignored when `hosts` is defined. | **{check-circle}** Yes | `389` or `636` (for SSL) | | `hosts` (GitLab 14.7 and later) | An array of host and port pairs to open connections. | **{dotted-circle}** No | `[['ldap1.mydomain.com', 636], ['ldap2.mydomain.com', 636]]` | -| `uid` | LDAP attribute for username. Should be the attribute, not the value that maps to the `uid`. | **{check-circle}** Yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | +| `uid` | The LDAP attribute that maps to the username that users use to sign in. Should be the attribute, not the value that maps to the `uid`. Does not affect the GitLab username (see [attributes section](#attribute-configuration-settings)). | **{check-circle}** Yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | | `bind_dn` | The full DN of the user you bind with. | **{dotted-circle}** No | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | | `password` | The password of the bind user. | **{dotted-circle}** No | `'your_great_password'` | | `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | **{check-circle}** Yes | `'start_tls'` or `'simple_tls'` or `'plain'` | diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 5c5d5aaffe8..c7572ec0a18 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -88,7 +88,7 @@ options = { # :filter is optional # This filter includes OID 1.2.840.113556.1.4.1941 - # It will search for all direct and nested members of the group gitlab_grp in the LDAP directory + # It will search for all direct and nested members of the group gitlab_grp in the LDAP directory filter: Net::LDAP::Filter.construct("(memberOf:1.2.840.113556.1.4.1941:=CN=gitlab_grp,DC=example,DC=com)"), # :attributes is optional @@ -707,7 +707,7 @@ For more information, see the [official `ldapsearch` documentation](https://linu ### Using **AdFind** (Windows) -You can use the [`AdFind`](https://social.technet.microsoft.com/wiki/contents/articles/7535.adfind-command-examples.aspx) utility (on Windows based systems) to test that your LDAP server is accessible and authentication is working correctly. AdFind is a freeware utility built by [Joe Richards](http://www.joeware.net/freetools/tools/adfind/index.htm). +You can use the [`AdFind`](https://social.technet.microsoft.com/wiki/contents/articles/7535.adfind-command-examples.aspx) utility (on Windows based systems) to test that your LDAP server is accessible and authentication is working correctly. AdFind is a freeware utility built by [Joe Richards](https://www.joeware.net/freetools/tools/adfind/index.htm). **Return all objects** diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 561cbd1b3ae..deb47160d98 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -120,7 +120,7 @@ The OpenID Connect provides you with a client's details and secret for you to us for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be an OpenID Connect icon below the regular sign in form. -Click the icon to begin the authentication process. The OpenID Connect provider asks the user to +Select the icon to begin the authentication process. The OpenID Connect provider asks the user to sign in and authorize the GitLab application (if confirmation required by the client). If everything goes well, the user is redirected to GitLab and signed in. diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index 3f0d95967bf..0590410bf31 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -120,8 +120,14 @@ more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/ 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby + # Allow smartcard authentication gitlab_rails['smartcard_enabled'] = true + + # Path to a file containing a CA certificate gitlab_rails['smartcard_ca_file'] = "/etc/ssl/certs/CA.pem" + + # Host and port where the client side certificate is requested by the + # webserver (NGINX/Apache) gitlab_rails['smartcard_client_certificate_required_host'] = "smartcard.example.com" gitlab_rails['smartcard_client_certificate_required_port'] = 3444 ``` diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 99c291379f1..869de929eb8 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -6,12 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Compliance features **(FREE)** -These GitLab features can help ensure that your GitLab instance meets common -compliance standards. For more information about compliance management, see the -compliance management [solutions page](https://about.gitlab.com/solutions/compliance/). +GitLab compliance features ensure your GitLab instance meets common compliance standards, and are available at various pricing tiers. For more information about compliance management, see the compliance +management [solutions page](https://about.gitlab.com/solutions/compliance/). -The [security features](../security/index.md) in GitLab may also help you meet -relevant compliance standards. +The [security features](../security/index.md) in GitLab may also help you meet relevant compliance standards. ## Policy management diff --git a/doc/administration/configure.md b/doc/administration/configure.md index cba4a23d95a..4335c6d972f 100644 --- a/doc/administration/configure.md +++ b/doc/administration/configure.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -46,5 +46,5 @@ The following table outlines failure modes and mitigation paths for the product | Single Gitaly Node + Geo Secondary | Downtime - Must restore from backup, can perform a manual failover to secondary | Downtime - Must restore from Backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | | Sharded Gitaly Install | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Downtime - Must wait for outage to end | | | Sharded Gitaly Install + Geo Secondary | Partial Downtime - Only repositories on impacted node affected, must restore from backup, could perform manual failover to secondary for impacted repositories | Partial Downtime - Only repositories on impacted node affected, must restore from backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | -| Gitaly Cluster Install* | No Downtime - will swap repository primary to another node after 10 seconds | N/A - All writes are voted on by multiple Gitaly Cluster nodes | Downtime - Must wait for outage to end | Snapshot backups for Gitaly Cluster nodes not supported at this time | -| Gitaly Cluster Install* + Geo Secondary | No Downtime - will swap repository primary to another node after 10 seconds | N/A - All writes are voted on by multiple Gitaly Cluster nodes | Manual intervention - failover to Geo secondary | Snapshot backups for Gitaly Cluster nodes not supported at this time | +| Gitaly Cluster Install* | No Downtime - will swap repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Downtime - Must wait for outage to end | Snapshot backups for Gitaly Cluster nodes not supported at this time | +| Gitaly Cluster Install* + Geo Secondary | No Downtime - will swap repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Manual intervention - failover to Geo secondary | Snapshot backups for Gitaly Cluster nodes not supported at this time | diff --git a/doc/administration/consul.md b/doc/administration/consul.md index 72b3487a549..e1f2bd90e05 100644 --- a/doc/administration/consul.md +++ b/doc/administration/consul.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -161,7 +161,7 @@ sudo gitlab-ctl restart consul ### Consul nodes unable to communicate By default, Consul attempts to -[bind](https://www.consul.io/docs/agent/options#_bind) to `0.0.0.0`, but +[bind](https://www.consul.io/docs/agent/config/config-files#bind_addr) to `0.0.0.0`, but it advertises the first private IP address on the node for other Consul nodes to communicate with it. If the other nodes cannot communicate with a node on this address, then the cluster has a failed status. diff --git a/doc/administration/docs_self_host.md b/doc/administration/docs_self_host.md index 8464c35c3bb..338601a054f 100644 --- a/doc/administration/docs_self_host.md +++ b/doc/administration/docs_self_host.md @@ -1,14 +1,19 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# How to host the GitLab product documentation **(FREE SELF)** +# Host the GitLab product documentation **(FREE SELF)** If you are not able to access the GitLab product documentation at `docs.gitlab.com`, you can host the documentation yourself instead. +Prerequisites: + +- The version of the product documentation site must be the same as the version of + your GitLab installation. + ## Documentation self-hosting options To host the GitLab product documentation, you can use: @@ -30,28 +35,48 @@ The following examples use GitLab 14.5. ### Self-host the product documentation with Docker -You can run the GitLab product documentation website in a Docker container: +The documentation website is served under the port `4000` inside the container. +In the following example, we expose this on the host under the same port. + +Make sure you either: + +- Allow port `4000` in your firewall. +- Use a different port. In following examples, replace the leftmost `4000` with the port different port. + +To run the GitLab product documentation website in a Docker container: -1. Expose port `4000`. The Docker image uses this port for the web server. 1. On the server where you host GitLab, or on any other server that your GitLab instance - can communicate with, pull the docs site: + can communicate with: - ```shell - docker run -it --rm -p 4000:4000 registry.gitlab.com/gitlab-org/gitlab-docs:14.5 - ``` + - If you use plain Docker, run: - If you host your GitLab instance using [Docker compose](../install/docker.md#install-gitlab-using-docker-compose), - add the following to `docker-compose.yaml`: + ```shell + docker run --detach --name gitlab_docs -it --rm -p 4000:4000 registry.gitlab.com/gitlab-org/gitlab-docs:14.5 + ``` - ```yaml - version: '3.6' - services: - docs: - image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5 - hostname: 'https://gitlab.example.com' - ports: - - '4000:4000' - ``` + - If you host your GitLab instance using + [Docker compose](../install/docker.md#install-gitlab-using-docker-compose), + add the following to your existing `docker-compose.yaml`: + + ```yaml + version: '3.6' + services: + gitlab_docs: + image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5 + hostname: 'https://docs.gitlab.example.com:4000' + ports: + - '4000:4000' + ``` + + Then, pull the changes: + + ```shell + docker-compose up -d + ``` + +1. Visit `http://0.0.0.0:4000` to view the documentation website and verify + it works. +1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). ### Self-host the product documentation with GitLab Pages @@ -89,29 +114,59 @@ To host the product documentation site with GitLab Pages: | [Project website](../user/project/pages/getting_started_part_one.md#project-website-examples) | Not supported | Supported | | [User or group website](../user/project/pages/getting_started_part_one.md#user-and-group-website-examples) | Supported | Supported | +1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). + ### Self-host the product documentation on your own web server -Because the product documentation site is static, from the container, you can take the contents -of `/usr/share/nginx/html` and use your own web server to host +Because the product documentation site is static, you can take the contents of +`/usr/share/nginx/html` from inside the container, and use your own web server to host the docs wherever you want. -Run the following commands, replacing `<destination>` with the directory where the -documentation files will be copied to: +The `html` directory should be served as is and it has the following structure: -```shell -docker create -it --name gitlab-docs registry.gitlab.com/gitlab-org/gitlab-docs:14.5 -docker cp gitlab-docs:/usr/share/nginx/html <destination> -docker rm -f gitlab-docs +```plaintext +├── 14.5/ +├── index.html ``` -## Redirect the `/help` links to the new docs page +In this example: + +- `14.5/` is the directory where the documentation is hosted. +- `index.html` is a simple HTML file that redirects to the directory containing the documentation. In this + case, `14.5/`. + +To extract the HTML files of the Docs site: + +1. Create the container that holds the HTML files of the documentation website: + + ```shell + docker create -it --name gitlab_docs registry.gitlab.com/gitlab-org/gitlab-docs:14.5 + ``` + +1. Copy the website under `/srv/gitlab/`: + + ```shell + docker cp gitlab-docs:/usr/share/nginx/html /srv/gitlab/ + ``` + + You will end up with a `/srv/gitlab/html/` directory that holds the documentation website. + +1. Remove the container: + + ```shell + docker rm -f gitlab_docs + ``` + +1. Point your web server to serve the contents of `/srv/gitlab/html/`. +1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). + +## Redirect the `/help` links to the new Docs site After your local product documentation site is running, [redirect the help links](../user/admin_area/settings/help_page.md#redirect-help-pages) -in the GitLab application to your local site. - -Be sure to use the fully qualified domain name as the docs URL. For example, if you -used the [Docker method](#self-host-the-product-documentation-with-docker), enter `http://0.0.0.0:4000`. +in the GitLab application to your local site, by using the fully qualified domain +name as the docs URL. For example, if you used the +[Docker method](#self-host-the-product-documentation-with-docker), enter `http://0.0.0.0:4000`. You don't need to append the version. GitLab detects it and appends it to documentation URL requests as needed. For example, if your GitLab version is @@ -124,6 +179,84 @@ documentation URL requests as needed. For example, if your GitLab version is To test the setting, select a **Learn more** link within the GitLab application. +## Upgrade the product documentation to a later version + +Upgrading the Docs site to a later version requires downloading the newer Docker image tag. + +### Upgrade using Docker + +To upgrade to a later version [using Docker](#self-host-the-product-documentation-with-docker): + +- If you use plain Docker: + + 1. Stop the running container: + + ```shell + sudo docker stop gitlab_docs + ``` + + 1. Remove the existing container: + + ```shell + sudo docker rm gitlab_docs + ``` + + 1. Pull the new image. For example, 14.6: + + ```shell + docker run --detach --name gitlab_docs -it --rm -p 4000:4000 registry.gitlab.com/gitlab-org/gitlab-docs:14.6 + ``` + +- If you use Docker compose: + + 1. Change the version in `docker-compose.yaml`, for example 14.6: + + ```yaml + version: '3.6' + services: + gitlab_docs: + image: registry.gitlab.com/gitlab-org/gitlab-docs:14.6 + hostname: 'https://docs.gitlab.example.com:4000' + ports: + - '4000:4000' + ``` + + 1. Pull the changes: + + ```shell + docker-compose up -d + ``` + +### Upgrade using GitLab Pages + +To upgrade to a later version [using GitLab Pages](#self-host-the-product-documentation-with-gitlab-pages): + +1. Edit your existing `.gitlab-ci.yml` file, and replace the `image`'s version number: + + ```yaml + image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5 + ``` + +1. Commit the changes, push, and GitLab Pages pulls the new Docs site version. + +### Upgrade using your own web-server + +To upgrade to a later version [using your own web-server](#self-host-the-product-documentation-on-your-own-web-server): + +1. Copy the HTML files of the Docs site: + + ```shell + docker create -it --name gitlab_docs registry.gitlab.com/gitlab-org/gitlab-docs:14.6 + docker cp gitlab_docs:/usr/share/nginx/html /srv/gitlab/ + docker rm -f gitlab_docs + ``` + +1. Optional. Remove the old site: + + ```shell + rm -r /srv/gitlab/html/14.5/ + ``` + ## Known issues If you self-host the product documentation: diff --git a/doc/administration/encrypted_configuration.md b/doc/administration/encrypted_configuration.md index 5c943e56b98..38f033e260a 100644 --- a/doc/administration/encrypted_configuration.md +++ b/doc/administration/encrypted_configuration.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" type: reference diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index 1fe5b223f8b..3adb057daa4 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -25,7 +25,7 @@ You can use the following environment variables to override certain values: | `EXTERNAL_VALIDATION_SERVICE_TIMEOUT` | integer | Timeout, in seconds, for an [external CI/CD pipeline validation service](external_pipeline_validation.md). Default is `5`. | | `EXTERNAL_VALIDATION_SERVICE_URL` | string | URL to an [external CI/CD pipeline validation service](external_pipeline_validation.md). | | `EXTERNAL_VALIDATION_SERVICE_TOKEN` | string | The `X-Gitlab-Token` for authentication with an [external CI/CD pipeline validation service](external_pipeline_validation.md). | -| `GITLAB_CDN_HOST` | string | Sets the base URL for a CDN to serve static assets (for example, `//mycdnsubdomain.fictional-cdn.com`). | +| `GITLAB_CDN_HOST` | string | Sets the base URL for a CDN to serve static assets (for example, `https://mycdnsubdomain.fictional-cdn.com`). | | `GITLAB_EMAIL_DISPLAY_NAME` | string | The name used in the **From** field in emails sent by GitLab. | | `GITLAB_EMAIL_FROM` | string | The email address used in the **From** field in emails sent by GitLab. | | `GITLAB_EMAIL_REPLY_TO` | string | The email address used in the **Reply-To** field in emails sent by GitLab. | diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index a4ed287cc3b..06eb9ffc84e 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -54,7 +54,9 @@ required number of seconds. "required": [ "id", "path", - "created_at" + "created_at", + "shared_runners_enabled", + "group_runners_enabled" ], "properties": { "id": { "type": "integer" }, diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index aebd6eb1d4c..8a021c6d588 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments description: "GitLab administrator: enable and disable GitLab features deployed behind feature flags" diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index 4ddca58c774..ed5dbd83a8e 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -51,7 +51,7 @@ Follow the steps below to set up a custom hook: in any language, and ensure the 'shebang' at the top properly reflects the language type. For example, if the script is in Ruby the shebang will probably be `#!/usr/bin/env ruby`. -1. The data to the file hook is provided as JSON on STDIN. It is exactly the +1. The data to the file hook is provided as JSON on `STDIN`. It is exactly the same as for [system hooks](system_hooks.md). That's it! Assuming the file hook code is properly implemented, the hook fires diff --git a/doc/administration/geo/disaster_recovery/background_verification.md b/doc/administration/geo/disaster_recovery/background_verification.md index d7db48bb6cf..57f8664ba11 100644 --- a/doc/administration/geo/disaster_recovery/background_verification.md +++ b/doc/administration/geo/disaster_recovery/background_verification.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index 4e68cc39580..833b9a877e9 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -18,8 +18,8 @@ If you have any doubts about the consistency of the data on this site, we recomm ## Configure the former **primary** site to be a **secondary** site -Since the former **primary** site will be out of sync with the current **primary** site, the first step is to bring the former **primary** site up to date. Note, deletion of data stored on disk like -repositories and uploads will not be replayed when bringing the former **primary** site back +Since the former **primary** site is out of sync with the current **primary** site, the first step is to bring the former **primary** site up to date. Note, deletion of data stored on disk like +repositories and uploads is not replayed when bringing the former **primary** site back into sync, which may result in increased disk usage. Alternatively, you can [set up a new **secondary** GitLab instance](../setup/index.md) to avoid this. @@ -34,8 +34,8 @@ To bring the former **primary** site up to date: NOTE: If you [disabled the **primary** site permanently](index.md#step-2-permanently-disable-the-primary-site), - you need to undo those steps now. For Debian/Ubuntu you just need to run - `sudo systemctl enable gitlab-runsvdir`. For CentOS 6, you need to install + you need to undo those steps now. For distributions with systemd, such as Debian/Ubuntu/CentOS7+, you must run + `sudo systemctl enable gitlab-runsvdir`. For distributions without systemd, such as CentOS 6, you need to install the GitLab instance from scratch and set it up as a **secondary** site by following [Setup instructions](../setup/index.md). In this case, you don't need to follow the next step. diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index baece830318..e5b9a14cfbd 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -335,16 +335,14 @@ a **secondary** with only a single node. Instead, you must do this manually. #### Promoting a **secondary** site with an external PostgreSQL database running GitLab 14.5 and later -The `gitlab-ctl geo promote` command can be used in conjunction with -an external PostgreSQL database, but it can only perform changes on -a **secondary** PostgreSQL database managed by Omnibus. -You must promote the replica database associated with the **secondary** -site first. +The `gitlab-ctl geo promote` command can be used in conjunction with an external PostgreSQL database. +In this case, you must first manually promote the replica database associated +with the **secondary** site: 1. Promote the replica database associated with the **secondary** site. This sets the database to read-write. The instructions vary depending on where your database is hosted: - [Amazon RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Promote) - - [Azure PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal#stop-replication) + - [Azure PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/single-server/how-to-read-replicas-portal#stop-replication) - [Google Cloud SQL](https://cloud.google.com/sql/docs/mysql/replication/manage-replicas#promote-replica) - For other external PostgreSQL databases, save the following script in your secondary site, for example `/tmp/geo_promote.sh`, and modify the connection diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 57bad6177d9..7d8dd7d5d2a 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -143,7 +143,7 @@ If the **primary** site uses custom or self-signed TLS certificates to secure in ### Ensure Geo replication is up-to-date -The maintenance window won't end until Geo replication and verification is +The maintenance window does not end until Geo replication and verification is completely finished. To keep the window as short as possible, you should ensure these processes are close to 100% as possible during active use. @@ -201,7 +201,7 @@ be disabled on the **primary** site: ## Finish replicating and verifying all data NOTE: -GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary site statuses will appear to stop updating and become unhealthy. For more information, see [Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](../replication/troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which the Geo secondary site statuses appears to stop updating and become unhealthy. For more information, see [Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](../replication/troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). 1. If you are manually replicating any data not managed by Geo, trigger the final replication process now. diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index 59134df8e7f..46b2ccddefd 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index 25f6999ce43..da26023e4f9 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/glossary.md b/doc/administration/geo/glossary.md index c6b3f26dc67..c9d4ccacae3 100644 --- a/doc/administration/geo/glossary.md +++ b/doc/administration/geo/glossary.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index dc25be825ea..c5472e412d3 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -13,7 +13,7 @@ a warm-standby as part of a disaster recovery strategy. WARNING: Geo undergoes significant changes from release to release. Upgrades are -supported and [documented](#updating-geo), but you should ensure that you're +supported and [documented](#upgrading-geo), but you should ensure that you're using the right version of the documentation for your installation. Fetching large repositories can take a long time for teams located far from a single GitLab instance. @@ -53,8 +53,7 @@ Geo provides: ### Gitaly Cluster Geo should not be confused with [Gitaly Cluster](../gitaly/praefect.md). For more information about -the difference between Geo and Gitaly Cluster, see -[How does Gitaly Cluster compare to Geo?](../gitaly/faq.md#how-does-gitaly-cluster-compare-to-geo). +the difference between Geo and Gitaly Cluster, see [Comparison to Geo](../gitaly/index.md#comparison-to-geo). ## How it works @@ -234,9 +233,9 @@ After installing GitLab on the **secondary** sites and performing the initial co For information on configuring Geo, see [Geo configuration](replication/configuration.md). -### Updating Geo +### Upgrading Geo -For information on how to update your Geo sites to the latest GitLab version, see [Updating the Geo sites](replication/updating_the_geo_sites.md). +For information on how to update your Geo sites to the latest GitLab version, see [Upgrading the Geo sites](replication/upgrading_the_geo_sites.md). ### Pausing and resuming replication @@ -252,7 +251,7 @@ WARNING: Pausing and resuming of replication is only supported for Geo installations using an Omnibus GitLab-managed database. External databases are not supported. -In some circumstances, like during [upgrades](replication/updating_the_geo_sites.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. +In some circumstances, like during [upgrades](replication/upgrading_the_geo_sites.md) or a [planned failover](disaster_recovery/planned_failover.md), it is desirable to pause replication between the primary and secondary. Pausing and resuming replication is done via a command line tool from the node in the secondary site where the `postgresql` service is enabled. diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index adf89c24a4e..043b96478c9 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -384,7 +384,7 @@ when: ## Upgrading Geo -See the [updating the Geo sites document](updating_the_geo_sites.md). +See the [upgrading the Geo sites document](upgrading_the_geo_sites.md). ## Troubleshooting diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 83e207e7a8f..8a5aa935d11 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -26,7 +26,7 @@ verification methods: | Type | Feature / component | Replication method | Verification method | |:---------|:------------------------------------------------|:--------------------------------------|:-----------------------| | Database | Application data in PostgreSQL | Native | Native | -| Database | Redis | _N/A_ (*1*) | _N/A_ | +| Database | Redis | Not applicable (*1*) | Not applicable | | Database | Elasticsearch | Native | Native | | Database | SSH public keys | PostgreSQL Replication | PostgreSQL Replication | | Git | Project repository | Geo with Gitaly | Gitaly Checksum | @@ -203,8 +203,10 @@ successfully, you must replicate their data using some other means. |[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification was behind the feature flag `geo_merge_request_diff_verification`, removed in 14.7.| |[Versioned snippets](../../../user/snippets.md#versioned-snippets) | [**Yes** (13.7)](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [**Yes** (14.2)](https://gitlab.com/groups/gitlab-org/-/epics/2810) | No | Verification was implemented behind the feature flag `geo_snippet_repository_verification` in 13.11, and the feature flag was removed in 14.2. | |[GitLab Pages](../../pages/index.md) | [**Yes** (14.3)](https://gitlab.com/groups/gitlab-org/-/epics/589) | [**Yes**](#limitation-of-verification-for-files-in-object-storage) (14.6) | Via Object Storage provider if supported. Native Geo support (Beta). | Behind feature flag `geo_pages_deployment_replication`, enabled by default. Verification was behind the feature flag `geo_pages_deployment_verification`, removed in 14.7. | +|[Incident Metric Images](../../../operations/incident_management/incidents.md#metrics) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | | +|[Alert Metric Images](../../../operations/incident_management/alerts.md#metrics-tab) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/352326) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | | |[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | No | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. | -|[Elasticsearch integration](../../../integration/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | +|[Elasticsearch integration](../../../integration/advanced_search/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | |[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. | |[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md index 0fa469e57cd..5d4ae12c990 100644 --- a/doc/administration/geo/replication/disable_geo.md +++ b/doc/administration/geo/replication/disable_geo.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/replication/docker_registry.md b/doc/administration/geo/replication/docker_registry.md index 4004ef3c17f..855e33d9a51 100644 --- a/doc/administration/geo/replication/docker_registry.md +++ b/doc/administration/geo/replication/docker_registry.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -114,10 +114,10 @@ For each application and Sidekiq node on the **secondary** site: ```ruby gitlab_rails['geo_registry_replication_enabled'] = true - + # Primary registry's hostname and port, it will be used by # the secondary node to directly communicate to primary registry - gitlab_rails['geo_registry_replication_primary_api_url'] = 'https://primary.example.com:5050/' + gitlab_rails['geo_registry_replication_primary_api_url'] = 'https://primary.example.com:5050/' ``` 1. Reconfigure the node for the change to take effect: diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index 12b3b382bf7..bdf1771e8a8 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -32,10 +32,10 @@ To ensure that problems with pipelines (for example, syncs failing too many time the number of concurrent syncs falls below `repos_max_capacity` and there are no new projects waiting to be synced. Geo also has a checksum feature which runs a SHA256 sum across all the Git references to the SHA values. -If the refs don't match between the **primary** site and the **secondary** site, then the **secondary** site will mark that project as dirty and try to resync it. +If the refs don't match between the **primary** site and the **secondary** site, then the **secondary** site marks that project as dirty and try to resync it. So even if we have an outdated tracking database, the validation should activate and find discrepancies in the repository state and resync. -## Can I use Geo in a disaster recovery situation? +## Can you use Geo in a disaster recovery situation? Yes, but there are limitations to what we replicate (see [What data is replicated to a **secondary** site?](#what-data-is-replicated-to-a-secondary-site)). @@ -46,7 +46,7 @@ Read the documentation for [Disaster Recovery](../disaster_recovery/index.md). We currently replicate project repositories, LFS objects, generated attachments and avatars, and the whole database. This means user accounts, -issues, merge requests, groups, project data, and so on, will be available for +issues, merge requests, groups, project data, and so on, are available for query. For more details, see the [supported Geo data types](datatypes.md). @@ -69,6 +69,6 @@ That's totally fine. We use HTTP(s) to fetch repository changes from the **prima Yes. See [Docker Registry for a **secondary** site](docker_registry.md). -## Can I login to a secondary site? +## Can you login to a secondary site? Yes, but secondary sites receive all authentication data (like user accounts and logins) from the primary instance. This means you are re-directed to the primary for authentication and then routed back. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index e4432c41ff1..6540366bf7f 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/geo/replication/location_aware_git_url.md b/doc/administration/geo/replication/location_aware_git_url.md index 8689fac987f..18103211580 100644 --- a/doc/administration/geo/replication/location_aware_git_url.md +++ b/doc/administration/geo/replication/location_aware_git_url.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -57,34 +57,34 @@ routing configurations.  -1. Click the **Create traffic policy** button. +1. Select the **Create traffic policy** button.  -1. Fill in the **Policy Name** field with `Single Git Host` and click **Next**. +1. Fill in the **Policy Name** field with `Single Git Host` and select **Next**.  1. Leave **DNS type** as `A: IP Address in IPv4 format`. -1. Click **Connect to...** and select **Geolocation rule**. +1. Select **Connect to...** and select **Geolocation rule**.  1. For the first **Location**, leave it as `Default`. -1. Click **Connect to...** and select **New endpoint**. +1. Select **Connect to...** and select **New endpoint**. 1. Choose **Type** `value` and fill it in with `<your **primary** IP address>`. 1. For the second **Location**, choose `Europe`. -1. Click **Connect to...** and select **New endpoint**. +1. Select **Connect to...** and select **New endpoint**. 1. Choose **Type** `value` and fill it in with `<your **secondary** IP address>`.  -1. Click **Create traffic policy**. +1. Select **Create traffic policy**.  1. Fill in **Policy record DNS name** with `git`. -1. Click **Create policy records**. +1. Select **Create policy records**.  diff --git a/doc/administration/geo/replication/multiple_servers.md b/doc/administration/geo/replication/multiple_servers.md index 7b800817461..afa4f4eb552 100644 --- a/doc/administration/geo/replication/multiple_servers.md +++ b/doc/administration/geo/replication/multiple_servers.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/replication/object_storage.md b/doc/administration/geo/replication/object_storage.md index 06f25fa15af..9ea226fd93b 100644 --- a/doc/administration/geo/replication/object_storage.md +++ b/doc/administration/geo/replication/object_storage.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/replication/remove_geo_site.md b/doc/administration/geo/replication/remove_geo_site.md index b69dfe2e723..0d6715a93b7 100644 --- a/doc/administration/geo/replication/remove_geo_site.md +++ b/doc/administration/geo/replication/remove_geo_site.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -44,7 +44,7 @@ Once GitLab has been uninstalled from each node on the **secondary** site, the r ``` NOTE: - Using `gitlab-rails dbconsole` will not work, because managing replication slots requires superuser permissions. + Using `gitlab-rails dbconsole` does not work, because managing replication slots requires superuser permissions. 1. Find the name of the relevant replication slot. This is the slot that is specified with `--slot-name` when running the replicate command: `gitlab-ctl replicate-geo-database`. diff --git a/doc/administration/geo/replication/security_review.md b/doc/administration/geo/replication/security_review.md index df893298f85..bc08b7b71e7 100644 --- a/doc/administration/geo/replication/security_review.md +++ b/doc/administration/geo/replication/security_review.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 5a29c5a3c54..bb7dbef214b 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -398,7 +398,7 @@ where some queries never complete due to being canceled on every replication. These long-running queries are [planned to be removed in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/34269), but as a workaround, we recommend enabling -[hot_standby_feedback](https://www.postgresql.org/docs/10/hot-standby.html#HOT-STANDBY-CONFLICT). +[`hot_standby_feedback`](https://www.postgresql.org/docs/10/hot-standby.html#HOT-STANDBY-CONFLICT). This increases the likelihood of bloat on the **primary** node as it prevents `VACUUM` from removing recently-dead rows. However, it has been used successfully in production on GitLab.com. @@ -767,7 +767,7 @@ The appropriate action sometimes depends on the cause. For example, you can remo In some cases, a file may be determined to be of low value, and so it may be worth deleting the record. -Geo itself is an excellent mitigation for files missing on the primary. If a file disappears on the primary but it was already synced to the secondary, you can grab the secondary's file. In cases like this, the `File is not checksummable` error message will not occur on Geo secondary sites, and only the primary will log this error message. +Geo itself is an excellent mitigation for files missing on the primary. If a file disappears on the primary but it was already synced to the secondary, you can grab the secondary's file. In cases like this, the `File is not checksummable` error message does not occur on Geo secondary sites, and only the primary logs this error message. This problem is more likely to show up in Geo secondary sites which were set up long after the original GitLab site. In this case, Geo is only surfacing an existing problem. @@ -815,6 +815,20 @@ Gitlab::Geo.verification_enabled_replicator_classes.each do |klass| end ``` +### Message: curl 18 transfer closed with outstanding read data remaining & fetch-pack: unexpected disconnect while reading sideband packet + +Unstable networking conditions can cause Gitaly to fail when trying to fetch large repository +data from the primary site. This is more likely to happen if a repository has to be +replicated from scratch between sites. + +Geo retries several times, but if the transmission is consistently interrupted +by network hiccups, an alternative method such as `rsync` can be used to circumvent `git` and +create the initial copy of any repository that fails to be replicated by Geo. + +We recommend transferring each failing repository individually and checking for consistency +after each transfer. Follow the [single target `rsync` instructions](../../operations/moving_repositories.md#single-rsync-to-another-server) +to transfer each affected repository from the primary to the secondary site. + ## Fixing errors during a failover or when promoting a secondary to a primary node The following are possible error messages that might be encountered during failover or @@ -1104,9 +1118,9 @@ If using a load balancer, ensure that the load balancer's URL is set as the `ext ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -In GitLab 13.9 through GitLab 14.3, when [GitLab Maintenance Mode](../../maintenance_mode/index.md) is enabled, the status of Geo secondary sites will stop getting updated. After 10 minutes, the status changes to `Unhealthy`. +In GitLab 13.9 through GitLab 14.3, when [GitLab Maintenance Mode](../../maintenance_mode/index.md) is enabled, the status of Geo secondary sites stops getting updated. After 10 minutes, the status changes to `Unhealthy`. -Geo secondary sites will continue to replicate and verify data, and the secondary sites should still be usable. You can use the [Sync status Rake task](#sync-status-rake-task) to determine the actual status of a secondary site during Maintenance Mode. +Geo secondary sites continue to replicate and verify data, and the secondary sites should still be usable. You can use the [Sync status Rake task](#sync-status-rake-task) to determine the actual status of a secondary site during Maintenance Mode. This bug was [fixed in GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/292983). @@ -1129,7 +1143,7 @@ Geo::TrackingBase::SecondaryNotConfigured: Geo secondary database is not configu On a Geo primary site this error can be ignored. -This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). +This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). ## Fixing client errors @@ -1148,7 +1162,7 @@ The partial failover to a secondary Geo *site* may be the result of a temporary/ 1. SSH into every Sidekiq, PostgresSQL, Gitaly, and Rails node in the **secondary** site and run one of the following commands: - To promote the secondary node to primary: - + ```shell sudo gitlab-ctl geo promote ``` @@ -1158,8 +1172,8 @@ The partial failover to a secondary Geo *site* may be the result of a temporary/ ```shell sudo gitlab-ctl geo promote --force ``` - -1. Verify you can connect to the newly-promoted **primary** site using the URL used previously for the **secondary** site. + +1. Verify you can connect to the newly-promoted **primary** site using the URL used previously for the **secondary** site. 1. If **successful**, the **secondary** site is now promoted to the **primary** site. If the above steps are **not successful**, proceed through the next steps: diff --git a/doc/administration/geo/replication/tuning.md b/doc/administration/geo/replication/tuning.md index bcff6181296..670459624f3 100644 --- a/doc/administration/geo/replication/tuning.md +++ b/doc/administration/geo/replication/tuning.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -7,7 +7,7 @@ type: howto # Tuning Geo **(PREMIUM SELF)** -You can limit the number of concurrent operations the nodes can run +You can limit the number of concurrent operations the sites can run in the background. ## Changing the sync/verification concurrency values @@ -15,8 +15,8 @@ in the background. On the **primary** site: 1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Geo > Nodes**. -1. Select **Edit** of the secondary node you want to tune. +1. On the left sidebar, select **Geo > Sites**. +1. Select **Edit** of the secondary site you want to tune. 1. Under **Tuning settings**, there are several variables that can be tuned to improve the performance of Geo: @@ -25,7 +25,7 @@ On the **primary** site: - Container repositories synchronization concurrency limit - Verification concurrency limit -Increasing the concurrency values will increase the number of jobs that are scheduled. +Increasing the concurrency values increases the number of jobs that are scheduled. However, this may not lead to more downloads in parallel unless the number of available Sidekiq threads is also increased. For example, if repository synchronization concurrency is increased from 25 to 50, you may also want to increase the number diff --git a/doc/administration/geo/replication/updating_the_geo_sites.md b/doc/administration/geo/replication/upgrading_the_geo_sites.md index e82afe5f0d4..30961de0381 100644 --- a/doc/administration/geo/replication/updating_the_geo_sites.md +++ b/doc/administration/geo/replication/upgrading_the_geo_sites.md @@ -1,31 +1,31 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto --- -# Updating the Geo sites **(PREMIUM SELF)** +# Upgrading the Geo sites **(PREMIUM SELF)** WARNING: Read these sections carefully before updating your Geo sites. Not following -version-specific update steps may result in unexpected downtime. If you have +version-specific upgrade steps may result in unexpected downtime. If you have any specific questions, [contact Support](https://about.gitlab.com/support/#contact-support). -Updating Geo sites involves performing: +Upgrading Geo sites involves performing: -1. [Version-specific update steps](version_specific_updates.md), depending on the - version being updated to or from. -1. [General update steps](#general-update-steps), for all updates. +1. [Version-specific upgrade steps](version_specific_upgrades.md), depending on the + version being upgraded to or from. +1. [General upgrade steps](#general-upgrade-steps), for all upgrades. -## General update steps +## General upgrade steps NOTE: -These general update steps are not intended for multi-site deployments, -and will cause downtime. If you want to avoid downtime, consider using +These general upgrade steps are not intended for multi-site deployments, +and cause downtime. If you want to avoid downtime, consider using [zero downtime upgrades](../../../update/zero_downtime.md#multi-node--ha-deployment-with-geo). -To update the Geo sites when a new GitLab version is released, update **primary** +To upgrade the Geo sites when a new GitLab version is released, upgrade **primary** and all **secondary** sites: 1. **Optional:** [Pause replication on each **secondary** sites.](../index.md#pausing-and-resuming-replication) @@ -34,11 +34,11 @@ and all **secondary** sites: 1. SSH into each node of **secondary** sites. 1. [Upgrade GitLab on each **secondary** site](../../../update/package/index.md#upgrade-using-the-official-repositories). 1. If you paused replication in step 1, [resume replication on each **secondary**](../index.md#pausing-and-resuming-replication) -1. [Test](#check-status-after-updating) **primary** and **secondary** sites, and check version in each. +1. [Test](#check-status-after-upgrading) **primary** and **secondary** sites, and check version in each. -### Check status after updating +### Check status after upgrading -Now that the update process is complete, you may want to check whether +Now that the upgrade process is complete, you may want to check whether everything is working correctly: 1. Run the Geo Rake task on an application node for the primary and secondary sites. Everything should be green: diff --git a/doc/administration/geo/replication/usage.md b/doc/administration/geo/replication/usage.md index d96bfae78a1..fe0e06e7ea4 100644 --- a/doc/administration/geo/replication/usage.md +++ b/doc/administration/geo/replication/usage.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -11,9 +11,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w After you set up the [database replication and configure the Geo nodes](../index.md#setup-instructions), use your closest GitLab site as you would do with the primary one. You can push directly to a **secondary** site (for both HTTP, SSH including -Git LFS), and the request will be proxied to the primary site instead. +Git LFS), and the request is proxied to the primary site instead. -Example of the output you will see when pushing to a **secondary** site: +Example of the output you see when pushing to a **secondary** site: ```shell $ git push @@ -31,7 +31,7 @@ If you're using HTTPS instead of [SSH](../../../user/ssh.md) to push to the seco you can't store credentials in the URL like `user:password@URL`. Instead, you can use a [`.netrc` file](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html) for Unix-like operating systems or `_netrc` for Windows. In that case, the credentials -will be stored as a plain text. If you're looking for a more secure way to store credentials, +are stored as a plain text. If you're looking for a more secure way to store credentials, you can use [Git Credential Storage](https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage). ## Fetch Go modules from Geo secondary sites diff --git a/doc/administration/geo/replication/version_specific_updates.md b/doc/administration/geo/replication/version_specific_upgrades.md index 6b617a21be8..f0925bdf87e 100644 --- a/doc/administration/geo/replication/version_specific_updates.md +++ b/doc/administration/geo/replication/version_specific_upgrades.md @@ -1,24 +1,24 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Version-specific update instructions **(PREMIUM SELF)** +# Version-specific upgrade instructions **(PREMIUM SELF)** -Review this page for update instructions for your version. These steps -accompany the [general steps](updating_the_geo_sites.md#general-update-steps) -for updating Geo sites. +Review this page for upgrade instructions for your version. These steps +accompany the [general steps](upgrading_the_geo_sites.md#general-upgrade-steps) +for upgrading Geo sites. -## Updating to 14.9 +## Upgrading to 14.9 -**DO NOT** update to GitLab 14.9.0. Instead, use 14.9.1 or later. +**Do not** upgrade to GitLab 14.9.0. Instead, use 14.9.1 or later. We've discovered an issue with Geo's CI verification feature that may [cause job traces to be lost](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6664). This issue was fixed in [the GitLab 14.9.1 patch release](https://about.gitlab.com/releases/2022/03/23/gitlab-14-9-1-released/). -If you have already updated to GitLab 14.9.0, you can disable the feature causing the issue by [disabling the `geo_job_artifact_replication` feature flag](../../feature_flags.md#how-to-enable-and-disable-features-behind-flags). +If you have already upgraded to GitLab 14.9.0, you can disable the feature causing the issue by [disabling the `geo_job_artifact_replication` feature flag](../../feature_flags.md#how-to-enable-and-disable-features-behind-flags). -## Updating to 14.2 through 14.7 +## Upgrading to 14.2 through 14.7 There is [an issue in GitLab 14.2 through 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/299819#note_822629467) that affects Geo when the GitLab-managed object storage replication is used, causing blob object types to fail synchronization. @@ -33,11 +33,11 @@ results in a loop that consistently fails for all objects stored in object stora For information on how to fix this, see [Troubleshooting - Failed syncs with GitLab-managed object storage replication](troubleshooting.md#failed-syncs-with-gitlab-managed-object-storage-replication). -## Updating to 14.4 +## Upgrading to 14.4 There is [an issue in GitLab 14.4.0 through 14.4.2](../../../update/index.md#1440) that can affect Geo and other features that rely on cronjobs. We recommend upgrading to GitLab 14.4.3 or later. -## Updating to 14.1, 14.2, 14.3 +## Upgrading to 14.1, 14.2, 14.3 ### Multi-arch images @@ -72,13 +72,13 @@ Otherwise, for each **secondary** site, on a Rails application node, open a [Rai end ``` -If you are running a version prior to 14.1 and are using Geo and multi-arch containers in your Container Registry, we recommend [upgrading](updating_the_geo_sites.md) to at least GitLab 14.1. +If you are running a version prior to 14.1 and are using Geo and multi-arch containers in your Container Registry, we recommend [upgrading](upgrading_the_geo_sites.md) to at least GitLab 14.1. ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 14.0/14.1 +## Upgrading to GitLab 14.0/14.1 ### Primary sites can not be removed from the UI @@ -90,13 +90,13 @@ If you are running an affected version and need to remove your Primary site, you ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 13.12 +## Upgrading to GitLab 13.12 -### Secondary sites re-download all LFS files upon update +### Secondary sites re-download all LFS files upon upgrade -We found an issue where [secondary sites re-download all LFS files](https://gitlab.com/gitlab-org/gitlab/-/issues/334550) upon update. This bug: +We found an issue where [secondary sites re-download all LFS files](https://gitlab.com/gitlab-org/gitlab/-/issues/334550) upon upgrade. This bug: - Only applies to Geo secondary sites that have replicated LFS objects. - Is _not_ a data loss risk. @@ -104,9 +104,9 @@ We found an issue where [secondary sites re-download all LFS files](https://gitl - May impact performance for GitLab installations with a large number of LFS files. If you don't have many LFS objects or can stand a bit of churn, then it is safe to let the secondary sites re-download LFS objects. -If you do have many LFS objects, or many Geo secondary sites, or limited bandwidth, or a combination of them all, then we recommend you skip GitLab 13.12.0 through 13.12.6 and update to GitLab 13.12.7 or newer. +If you do have many LFS objects, or many Geo secondary sites, or limited bandwidth, or a combination of them all, then we recommend you skip GitLab 13.12.0 through 13.12.6 and upgrade to GitLab 13.12.7 or newer. -#### If you have already updated to an affected version, and the re-sync is ongoing +#### If you have already upgraded to an affected version, and the re-sync is ongoing You can manually migrate the legacy sync state to the new state column by running the following command in a [Rails console](../../operations/rails_console.md). It should take under a minute: @@ -116,29 +116,29 @@ Geo::LfsObjectRegistry.where(state: 0, success: true).update_all(state: 2) ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 13.11 +## Upgrading to GitLab 13.11 We found an [issue with Git clone/pull through HTTP(s)](https://gitlab.com/gitlab-org/gitlab/-/issues/330787) on Geo secondaries and on any GitLab instance if maintenance mode is enabled. This was caused by a regression in GitLab Workhorse. This is fixed in the [GitLab 13.11.4 patch release](https://about.gitlab.com/releases/2021/05/14/gitlab-13-11-4-released/). To avoid this issue, upgrade to GitLab 13.11.4 or later. ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 13.10 +## Upgrading to GitLab 13.10 ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 13.9 +## Upgrading to GitLab 13.9 -### Error during zero-downtime update: "cannot drop column asset_proxy_whitelist" +### Error during zero-downtime upgrade: "cannot drop column asset_proxy_whitelist" We've detected an issue [with a column rename](https://gitlab.com/gitlab-org/gitlab/-/issues/324160) that prevents upgrades to GitLab 13.9.0, 13.9.1, 13.9.2 and 13.9.3 when following the zero-downtime steps. It is necessary -to perform the following additional steps for the zero-downtime update: +to perform the following additional steps for the zero-downtime upgrade: 1. Before running the final `sudo gitlab-rake db:migrate` command on the deploy node, execute the following queries using the PostgreSQL console (or `sudo gitlab-psql`) @@ -169,27 +169,27 @@ PG::DependentObjectsStillExist: ERROR: cannot drop column asset_proxy_whitelist DETAIL: trigger trigger_0d588df444c8 on table application_settings depends on column asset_proxy_whitelist of table application_settings ``` -To work around this bug, follow the previous steps to complete the update. +To work around this bug, follow the previous steps to complete the upgrade. More details are available [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/324160). ### Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode -GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop updating and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). +GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab Maintenance Mode](../../maintenance_mode/index.md) causes Geo secondary site statuses to appear to stop upgrading and become unhealthy. For more information, see [Troubleshooting - Geo Admin Area shows 'Unhealthy' after enabling Maintenance Mode](troubleshooting.md#geo-admin-area-shows-unhealthy-after-enabling-maintenance-mode). -## Updating to GitLab 13.7 +## Upgrading to GitLab 13.7 We've detected an issue with the `FetchRemove` call used by Geo secondaries. This causes performance issues as we execute reference transaction hooks for -each updated reference. Delay any upgrade attempts until this is in the +each upgraded reference. Delay any upgrade attempts until this is in the [13.7.5 patch release.](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3002). More details are available [in this issue](https://gitlab.com/gitlab-org/git/-/issues/79). -## Updating to GitLab 13.5 +## Upgrading to GitLab 13.5 GitLab 13.5 has a [regression that prevents viewing a list of container repositories and registries](https://gitlab.com/gitlab-org/gitlab/-/issues/285475) on Geo secondaries. This issue is fixed in GitLab 13.6.1 and later. -## Updating to GitLab 13.3 +## Upgrading to GitLab 13.3 In GitLab 13.3, Geo removed the PostgreSQL [Foreign Data Wrapper](https://www.postgresql.org/docs/11/postgres-fdw.html) dependency for the tracking database. @@ -219,61 +219,61 @@ when using `--force` or `--skip-preflight-checks`, due to [an issue](https://git The [troubleshooting steps](troubleshooting.md#errors-when-using---skip-preflight-checks-or---force) contain a workaround if you run into errors during the failover. -## Updating to GitLab 13.2 +## Upgrading to GitLab 13.2 In GitLab 13.2, promoting a secondary site to a primary while the secondary is paused fails. Do not pause replication before promoting a secondary. If the site is paused, be sure to resume before promoting. To avoid this issue, upgrade to GitLab 13.4 or later. -## Updating to GitLab 13.0 +## Upgrading to GitLab 13.0 Upgrading to GitLab 13.0 requires GitLab 12.10 to already be using PostgreSQL version 11. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -## Updating to GitLab 12.10 +## Upgrading to GitLab 12.10 -GitLab 12.10 doesn't attempt to update the embedded PostgreSQL server when +GitLab 12.10 doesn't attempt to upgrade the embedded PostgreSQL server when using Geo, because the PostgreSQL upgrade requires downtime for secondaries while reinitializing streaming replication. It must be upgraded manually. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -## Updating to GitLab 12.9 +## Upgrading to GitLab 12.9 WARNING: GitLab 12.9.0 through GitLab 12.9.3 are affected by [a bug that stops repository verification](https://gitlab.com/gitlab-org/gitlab/-/issues/213523). The issue is fixed in GitLab 12.9.4. Upgrade to GitLab 12.9.4 or later. -By default, GitLab 12.9 attempts to update the embedded PostgreSQL server +By default, GitLab 12.9 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.12, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.8 +## Upgrading to GitLab 12.8 -By default, GitLab 12.8 attempts to update the embedded PostgreSQL server +By default, GitLab 12.8 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.12, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.7 +## Upgrading to GitLab 12.7 WARNING: Only upgrade to GitLab 12.7.5 or later. Do not upgrade to versions 12.7.0 @@ -281,65 +281,65 @@ through 12.7.4 because there is [an initialization order bug](https://gitlab.com [The fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24021) was shipped in 12.7.5. -By default, GitLab 12.7 attempts to update the embedded PostgreSQL server +By default, GitLab 12.7 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.6 +## Upgrading to GitLab 12.6 -By default, GitLab 12.6 attempts to update the embedded PostgreSQL server +By default, GitLab 12.6 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.5 +## Upgrading to GitLab 12.5 -By default, GitLab 12.5 attempts to update the embedded PostgreSQL server +By default, GitLab 12.5 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.4 +## Upgrading to GitLab 12.4 -By default, GitLab 12.4 attempts to update the embedded PostgreSQL server +By default, GitLab 12.4 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9, which requires downtime on secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). You can temporarily disable this behavior by running the following before -updating: +upgrading: ```shell sudo touch /etc/gitlab/disable-postgresql-upgrade ``` -## Updating to GitLab 12.3 +## Upgrading to GitLab 12.3 WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to -GitLab 12.4 or later. By default, GitLab 12.3 attempts to update the embedded +GitLab 12.4 or later. By default, GitLab 12.3 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can fail. For more information, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). @@ -349,11 +349,11 @@ requires downtime for secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -## Updating to GitLab 12.2 +## Upgrading to GitLab 12.2 WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to -GitLab 12.4 or later. By default, GitLab 12.2 attempts to update the embedded +GitLab 12.4 or later. By default, GitLab 12.2 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can fail. For more information, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). @@ -363,12 +363,12 @@ requires downtime for secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -GitLab 12.2 includes the following minor PostgreSQL updates: +GitLab 12.2 includes the following minor PostgreSQL upgrades: - To version `9.6.14`, if you run PostgreSQL 9.6. - To version `10.9`, if you run PostgreSQL 10. -This update occurs even if major PostgreSQL updates are disabled. +This upgrade occurs even if major PostgreSQL upgrades are disabled. Before [refreshing Foreign Data Wrapper during a Geo upgrade](../../../update/zero_downtime.md#step-4-run-post-deployment-migrations-and-checks), restart the Geo tracking database: @@ -380,11 +380,11 @@ sudo gitlab-ctl restart geo-postgresql The restart avoids a version mismatch when PostgreSQL tries to load the FDW extension. -## Updating to GitLab 12.1 +## Upgrading to GitLab 12.1 WARNING: If the existing PostgreSQL server version is 9.6.x, we recommend upgrading to -GitLab 12.4 or later. By default, GitLab 12.1 attempts to update the embedded +GitLab 12.4 or later. By default, GitLab 12.1 attempts to upgrade the embedded PostgreSQL server version from 9.6 to 10.9. In certain circumstances, it can fail. For more information, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). @@ -394,7 +394,7 @@ requires downtime for secondaries while reinitializing streaming replication. For the recommended procedure, see the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -## Updating to GitLab 12.0 +## Upgrading to GitLab 12.0 WARNING: This version is affected by a [bug that results in new LFS objects not being diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index 768adab9101..9a1aab8c238 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -10,6 +10,7 @@ type: howto > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5914) in GitLab 14.4 [with a flag](../../feature_flags.md) named `geo_secondary_proxy`. Disabled by default. > - [Enabled by default for unified URLs](https://gitlab.com/gitlab-org/gitlab/-/issues/325732) in GitLab 14.6. > - [Disabled by default for different URLs](https://gitlab.com/gitlab-org/gitlab/-/issues/325732) in GitLab 14.6 [with a flag](../../feature_flags.md) named `geo_secondary_proxy_separate_urls`. +> - [Enabled by default for different URLs](https://gitlab.com/gitlab-org/gitlab/-/issues/346112) in GitLab 15.1. FLAG: On self-managed GitLab, this feature is only available by default for Geo sites using a unified URL. See below to @@ -36,6 +37,9 @@ Use secondary proxying for use-cases including: - Having all Geo sites behind a single URL. - Geographically load-balancing traffic without worrying about write access. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview, see: [Secondary proxying using geographic load-balancer and AWS Route53](https://www.youtube.com/watch?v=TALLy7__Na8). + ## Set up a unified URL for Geo sites Secondary sites can transparently serve read-write traffic. You can @@ -104,17 +108,19 @@ gitlab: GEO_SECONDARY_PROXY: "0" ``` -## Enable Geo proxying with Separate URLs +## Geo proxying with Separate URLs + +Since GitLab 15.1, Geo secondary proxying is enabled by default for separate URLs also. -The ability to use proxying with separate URLs is still in development. You can follow the -["Geo secondary proxying with separate URLs" epic](https://gitlab.com/groups/gitlab-org/-/epics/6865) -for progress. +There are minor known issues linked in the ["Geo secondary proxying with separate URLs" +epic](https://gitlab.com/groups/gitlab-org/-/epics/6865). You can also add feedback in the epic about any use-cases that +are not possible anymore with proxying enabled. -To try out this feature, enable the `geo_secondary_proxy_separate_urls` feature flag. +If you run into issues, to disable this feature, disable the `geo_secondary_proxy_separate_urls` feature flag. SSH into one node running Rails on your primary Geo site and run: ```shell -sudo gitlab-rails runner "Feature.enable(:geo_secondary_proxy_separate_urls)" +sudo gitlab-rails runner "Feature.disable(:geo_secondary_proxy_separate_urls)" ``` In Kubernetes, you can run the same command in the toolbox pod. Refer to the diff --git a/doc/administration/geo/secondary_proxy/location_aware_external_url.md b/doc/administration/geo/secondary_proxy/location_aware_external_url.md index b5a8d4baa1f..1430e99557f 100644 --- a/doc/administration/geo/secondary_proxy/location_aware_external_url.md +++ b/doc/administration/geo/secondary_proxy/location_aware_external_url.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index 52f4adc4e03..7dfca4c8f73 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -177,10 +177,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o In most cases, the following addresses are used to configure GitLab Geo: - | Configuration | Address | - |:----------------------------------------|:------------------------------------------------------| - | `postgresql['listen_address']` | **Primary** node's public or VPC private address. | - | `postgresql['md5_auth_cidr_addresses']` | **Secondary** node's public or VPC private addresses. | + | Configuration | Address | + |:----------------------------------------|:----------------------------------------------------------------------| + | `postgresql['listen_address']` | **Primary** node's public or VPC private address. | + | `postgresql['md5_auth_cidr_addresses']` | **Primary** and **Secondary** nodes' public or VPC private addresses. | If you are using Google Cloud Platform, SoftLayer, or any other vendor that provides a virtual private cloud (VPC) you can use the **primary** and **secondary** nodes diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index 58fd6b28797..7e8ffa829f9 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -51,7 +51,7 @@ developed and tested. We aim to be compatible with most external gitlab-ctl set-geo-primary-node ``` - This command will use your defined `external_url` in `/etc/gitlab/gitlab.rb`. + This command uses your defined `external_url` in `/etc/gitlab/gitlab.rb`. ### Configure the external database to be replicated @@ -64,14 +64,14 @@ To set up an external database, you can either: Given you have a primary node set up on AWS EC2 that uses RDS. You can now just create a read-only replica in a different region and the -replication process will be managed by AWS. Make sure you've set Network ACL, Subnet, and +replication process is managed by AWS. Make sure you've set Network ACL (Access Control List), Subnet, and Security Group according to your needs, so the secondary application node can access the database. The following instructions detail how to create a read-only replica for common cloud providers: - Amazon RDS - [Creating a Read Replica](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ReadRepl.html#USER_ReadRepl.Create) -- Azure Database for PostgreSQL - [Create and manage read replicas in Azure Database for PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/howto-read-replicas-portal) +- Azure Database for PostgreSQL - [Create and manage read replicas in Azure Database for PostgreSQL](https://docs.microsoft.com/en-us/azure/postgresql/single-server/how-to-read-replicas-portal) - Google Cloud SQL - [Creating read replicas](https://cloud.google.com/sql/docs/postgres/replication/create-replica) Once your read-only replica is set up, you can skip to [configure your secondary application node](#configure-secondary-application-nodes-to-use-the-external-read-replica). @@ -190,7 +190,7 @@ to grant additional roles to your tracking database user (by default, this is `gitlab_geo`): - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. -- Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/howto-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. +- Azure Database for PostgreSQL requires the [`azure_pg_admin`](https://docs.microsoft.com/en-us/azure/postgresql/single-server/how-to-create-users#how-to-create-additional-admin-users-in-azure-database-for-postgresql) role. - Google Cloud SQL requires the [`cloudsqlsuperuser`](https://cloud.google.com/sql/docs/postgres/users#default-users) role. This is for the installation of extensions during installation and upgrades. As an alternative, diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md index 04a341aa822..f4c21293782 100644 --- a/doc/administration/geo/setup/index.md +++ b/doc/administration/geo/setup/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto @@ -19,7 +19,7 @@ The steps below should be followed in the order they appear. **Make sure the Git If you installed GitLab using the Omnibus packages (highly recommended): -1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the nodes that will serve as the **secondary** site. Do not create an account or log in to the new **secondary** site. +1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the nodes that serve as the **secondary** site. Do not create an account or log in to the new **secondary** site. The **GitLab version must match** across primary and secondary sites. 1. [Add the GitLab License](../../../user/admin_area/license.md) on the **primary** site to unlock Geo. The license must be for [GitLab Premium](https://about.gitlab.com/pricing/) or higher. 1. [Set up the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology). 1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** sites. diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 6888a2abe9a..26aa750900b 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -67,7 +67,7 @@ When running Gitaly on its own server, note the following regarding GitLab versi servers. - From GitLab 11.8 to 12.2, it is possible to use Elasticsearch in a Gitaly setup that doesn't use NFS. To use Elasticsearch in these versions, the - [repository indexer](../../integration/elasticsearch.md#elasticsearch-repository-indexer) + [repository indexer](../../integration/advanced_search/elasticsearch.md#elasticsearch-repository-indexer) must be enabled in your GitLab configuration. - [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/6481), the new indexer is the default and no configuration is required. @@ -825,14 +825,14 @@ information, see the [relevant documentation](monitoring.md#monitor-gitaly-concu ## Control groups -> - Introduced in GitLab 13.10. +> Introduced in GitLab 13.10. Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations, especially for large repositories. Control groups (cgroups) in Linux allow limits to be imposed on how much memory and CPU can be consumed. See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information. -cgroups can be useful for protecting the system against resource exhaustion because of overconsumption of memory and CPU. +cgroups can be useful for protecting the system against resource exhaustion because of overcomsumption of memory and CPU. Gitaly has built-in cgroups control. When configured, Gitaly assigns Git processes to a cgroup based on the repository the Git command is operating in. @@ -856,7 +856,59 @@ Using cgroups allows the kernel to kill these operations before they hog up all ### Configure cgroups in Gitaly -To configure cgroups in Gitaly, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. For +Two ways of configuring cgroups are available. + +#### Configure cgroups (new method) + +> This method of configuring cgroups introduced in GitLab 15.1. + +FLAG: +On self-managed GitLab, by default this method of configuring cgroups is not available. To make it available, ask an administrator to +[enable the feature flag](../feature_flags.md) named `gitaly_run_cmds_in_cgroup`. + +Gitaly creates a pool of cgroups that are isolated based on the repository used in the Git command to be placed under one of these cgroups. + +To configure cgroups in Gitaly, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. + +For example: + +```ruby +# in /etc/gitlab/gitlab.rb +gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" +gitaly['cgroups_hierarchy_root'] =>"gitaly" +gitaly['cgroups_memory_bytes'] = 64424509440, # 60gb +gitaly['cgroups_cpu_shares'] = 1024 +gitaly['cgroups_repositories_count'] => 1000, +gitaly['cgroups_repositories_memory_bytes'] => 32212254720 # 20gb +gitaly['cgroups_repositories_cpu_shares'] => 512 +``` + +- `cgroups_mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. +- `cgroups_hierarchy_root` is the parent cgroup under which Gitaly creates groups, and + is expected to be owned by the user and group Gitaly runs as. Omnibus GitLab + creates the set of directories `mountpoint/<cpu|memory>/hierarchy_root` + when Gitaly starts. +- `cgroups_memory_bytes` is the total memory limit that is imposed collectively on all + Git processes that Gitaly spawns. 0 implies no limit. +- `cgroups_cpu_shares` is the cpu limit that is imposed collectively on all Git + processes that Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, + which represents 100% of CPU. +- `cgroups_repositories_count` is the number of cgroups in the cgroups pool. Each time a new Git + command is spawned, Gitaly assigns it to one of these cgroups based + on the repository the command is for. A circular hashing algorithm assigns + Git commands to these cgroups, so a Git command for a repository is + always assigned to the same cgroup. +- `cgroups_repositories_memory_bytes` is the total memory limit that is imposed collectively on all + Git processes that Gitaly spawns. 0 implies no limit. This value cannot exceed + that of the top level `cgroups_memory_bytes`. +- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed collectively on all Git + processes Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, + which represents 100% of CPU. This value cannot exceed that of the top + level`cgroups_cpu_shares`. + +#### Configure cgroups (legacy method) + +To configure cgroups in Gitaly for GitLab versions using the legacy method, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. For example: ```ruby @@ -868,7 +920,6 @@ gitaly['cgroups_memory_limit'] = 32212254720 gitaly['cgroups_memory_enabled'] = true gitaly['cgroups_cpu_shares'] = 1024 gitaly['cgroups_cpu_enabled'] = true - ``` - `cgroups_count` is the number of cgroups created. Each time a new @@ -883,9 +934,28 @@ gitaly['cgroups_cpu_enabled'] = true - `cgroups_memory_enabled` enables or disables the memory limit on cgroups. - `cgroups_memory_bytes` is the total memory limit each cgroup imposes on the processes added to it. - `cgroups_cpu_enabled` enables or disables the CPU limit on cgroups. -- `cgroups_cpu_shares` is the CPU limit each cgroup imposes on the processes added to it. The maximum is 1024 shares, - which represents 100% of CPU. - which represents 100% of CPU. +- `cgroups_cpu_shares` is the CPU limit each cgroup imposes on the processes added to it. The maximum is 1024 shares, which represents 100% of CPU. + +### Configuring oversubscription + +In the previous example using the new configuration method: + +- The top level memory limit is capped at 60gb. +- Each of the 1000 cgroups in the repositories pool is capped at 20gb. + +This is called "oversubscription". Each cgroup in the pool has a much larger capacity than 1/1000th +of the top-level memory limit. + +This strategy has two main benefits: + +- It gives the host protection from overall memory starvation (OOM), because the top-level + cgroup's memory limit can be set to a threshold smaller than the host's + capacity. Processes outside of that cgroup are not at risk of OOM. +- It allows each individual cgroup in the pool to burst up to a generous upper + bound (in this example 20 GB) that is smaller than the parent cgroup's limit, + but substantially larger than 1/N of the parent's limit. In this example, up + to 3 child cgroups can concurrently burst up to their max. In general, all + 1000 cgroups would use much less than the 20 GB. ## Background Repository Optimization diff --git a/doc/administration/gitaly/faq.md b/doc/administration/gitaly/faq.md index a5c2c7d1469..f6571295200 100644 --- a/doc/administration/gitaly/faq.md +++ b/doc/administration/gitaly/faq.md @@ -1,90 +1,6 @@ --- -stage: Create -group: Gitaly -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-08-24' --- -# Frequently asked questions **(FREE SELF)** - -The following are answers to frequently asked questions about Gitaly and Gitaly Cluster. For -troubleshooting information, see [Troubleshooting Gitaly and Gitaly Cluster](troubleshooting.md). - -## How does Gitaly Cluster compare to Geo? - -Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: - -- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are - not aware when Gitaly Cluster is used. -- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for - an entire instance of GitLab. Users know when they are using Geo for - [replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification), - including Git data. - -The following table outlines the major differences between Gitaly Cluster and Geo: - -| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | -|:---------------|:---------|:----------|:------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------|:--------------------------------------|:------------------------| -| Gitaly Cluster | Multiple | Single | [Less than 1 second, ideally single-digit milliseconds](praefect.md#network-latency-and-connectivity) | [Automatic](praefect.md#automatic-failover-and-primary-election-strategies) | [Strong](index.md#strong-consistency) | Data storage in Git | -| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | - -For more information, see: - -- Geo [use cases](../geo/index.md#use-cases). -- Geo [architecture](../geo/index.md#architecture). - -## Are there instructions for migrating to Gitaly Cluster? - -Yes! For more information, see [Migrating to Gitaly Cluster](index.md#migrating-to-gitaly-cluster). - -## What are some repository storage recommendations? - -The size of the required storage can vary between instances and depends on the set -[replication factor](index.md#replication-factor). You might want to include implementing -repository storage redundancy. - -For a replication factor: - -- Of `1`: NFS, Gitaly, and Gitaly Cluster have roughly the same storage requirements. -- More than `1`: The amount of required storage is `used space * replication factor`. `used space` - should include any planned future growth. - -## What are some Praefect database storage requirements? - -The requirements are relatively low because the database contains only metadata of: - -- Where repositories are located. -- Some queued work. - -It depends on the number of repositories, but a useful minimum is 5-10 GB, similar to the main -GitLab application database. - -## Can the GitLab application database and the Praefect database be on the same servers? - -Yes, however Praefect should have it's own database server when using Omnibus GitLab PostgreSQL. If -there is a failover, Praefect isn't aware and starts to fail as the database it's trying to use would -either: - -- Be unavailable. -- In read-only mode. - -A future solution may allow for Praefect and Omnibus GitLab databases on the same PostgreSQL server. -For more information, see the relevant: - -- [Omnibus GitLab issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). -- [Gitaly issue](https://gitlab.com/gitlab-org/gitaly/-/issues/3398). - -## Is PgBouncer required for the Praefect database? - -No, because the number of connections Praefect makes is low. You can use the same PgBouncer instance -for both the GitLab application database and the Praefect database if you wish. - -## Are there any special considerations for Gitaly Cluster when PostgreSQL is upgraded? - -There are no special requirements. Gitaly Cluster requires PostgreSQL version 11 or later. - -## Praefect database tables are empty? - -These tables are created per the [specific configuration section](praefect.md#postgresql). - -If you find you have an empty Praefect database table, see the -[relevant troubleshooting section](troubleshooting.md#relation-does-not-exist-errors). +This document was moved to [another location](index.md). diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index f43f9f5d6c1..160313073a5 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -71,7 +71,7 @@ the current status of these issues, please refer to the referenced issues and ep | Issue | Summary | How to avoid | |:--------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------| | Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. Work is in-progress to update Gitaly Cluster to [identify repositories with a unique and persistent identifier](https://gitlab.com/gitlab-org/gitaly/-/issues/3485), which is expected to resolve the issue. | No known solution at this time. | -| Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform normal operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting live upgrade assistance](https://about.gitlab.com/support/scheduling-upgrade-assistance.html) so your upgrade plan can be reviewed by support. | +| Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform normal operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting live upgrade assistance](https://about.gitlab.com/support/scheduling-upgrade-assistance/) so your upgrade plan can be reviewed by support. | | Restoring a Gitaly Cluster node from a snapshot in a running cluster | Because the Gitaly Cluster runs with consistent state, introducing a single node that is behind will result in the cluster not being able to reconcile the nodes data and other nodes data | Don't restore a single Gitaly Cluster node from a backup snapshot. If you must restore from backup, it's best to snapshot all Gitaly Cluster nodes at the same time and take a database dump of the Praefect database. | ### Snapshot backup and recovery limitations @@ -206,6 +206,29 @@ WARNING: If complete cluster failure occurs, disaster recovery plans should be executed. These can affect the RPO and RTO discussed above. +### Comparison to Geo + +Gitaly Cluster and [Geo](../geo/index.md) both provide redundancy. However the redundancy of: + +- Gitaly Cluster provides fault tolerance for data storage and is invisible to the user. Users are + not aware when Gitaly Cluster is used. +- Geo provides [replication](../geo/index.md) and [disaster recovery](../geo/disaster_recovery/index.md) for + an entire instance of GitLab. Users know when they are using Geo for + [replication](../geo/index.md). Geo [replicates multiple data types](../geo/replication/datatypes.md#limitations-on-replicationverification), + including Git data. + +The following table outlines the major differences between Gitaly Cluster and Geo: + +| Tool | Nodes | Locations | Latency tolerance | Failover | Consistency | Provides redundancy for | +|:---------------|:---------|:----------|:------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------|:--------------------------------------|:------------------------| +| Gitaly Cluster | Multiple | Single | [Less than 1 second, ideally single-digit milliseconds](praefect.md#network-latency-and-connectivity) | [Automatic](praefect.md#automatic-failover-and-primary-election-strategies) | [Strong](index.md#strong-consistency) | Data storage in Git | +| Geo | Multiple | Multiple | Up to one minute | [Manual](../geo/disaster_recovery/index.md) | Eventual | Entire GitLab instance | + +For more information, see: + +- Geo [use cases](../geo/index.md#use-cases). +- Geo [architecture](../geo/index.md#architecture). + ### Virtual storage Virtual storage makes it viable to have a single repository storage in GitLab to simplify repository @@ -320,9 +343,9 @@ Use the [`praefect metadata`](troubleshooting.md#view-repository-metadata) subco - The virtual storage and relative path. - The repository ID. -The repository on disk also contains the project path in the Git configuration file. The configuration file can be used to determine -the project's location even if the repository's metadata has been deleted. Follow the -[instructions in hashed storage's documentation](../repository_storage_types.md#from-hashed-path-to-project-name). +The repository on disk also contains the project path in the Git configuration file. The configuration +file can be used to determine the project path even if the repository's metadata has been deleted. +Follow the [instructions in hashed storage's documentation](../repository_storage_types.md#from-hashed-path-to-project-name). #### Atomicity of operations @@ -330,7 +353,7 @@ Gitaly Cluster uses the PostgreSQL metadata store with the storage layout to ens deletion, and move operations. The disk operations can't be atomically applied across multiple storages. However, PostgreSQL guarantees the atomicity of the metadata operations. Gitaly Cluster models the operations in a manner that the failing operations always leave the metadata consistent. The disks may contain stale state even after successful operations. This is expected and the leftover state -won't intefere with future operations but may use up disk space unnecessarily until a clean up is performed. +won't interfere with future operations but may use up disk space unnecessarily until a clean up is performed. There is on-going work on a [background crawler](https://gitlab.com/gitlab-org/gitaly/-/issues/3719) that cleans up the leftover repositories from the storages. @@ -366,7 +389,7 @@ relative path of the repository in the metadata store. ### Moving beyond NFS Engineering support for NFS for Git repositories is deprecated. Technical support is planned to be unavailable starting -November 22, 2022. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) +November 22, 2022. Please see our [statement of support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) for more details. [Network File System (NFS)](https://en.wikipedia.org/wiki/Network_File_System) @@ -495,24 +518,31 @@ For configuration information, see [Configure replication factor](praefect.md#co For more information on configuring Gitaly Cluster, see [Configure Gitaly Cluster](praefect.md). -## Migrating to Gitaly Cluster +### Upgrade Gitaly Cluster + +To upgrade a Gitaly Cluster, follow the documentation for +[zero downtime upgrades](../../update/zero_downtime.md#gitaly-or-gitaly-cluster). -See the [Before deploying Gitaly Cluster](#before-deploying-gitaly-cluster) section before continuing. The basic process -for migrating to Gitaly Cluster involves: +## Migrate to Gitaly Cluster + +WARNING: +Some [known issues](#known-issues) exist in Gitaly Cluster. Review the following information before you continue. + +Before migrating to Gitaly Cluster: + +- Review [Before deploying Gitaly Cluster](#before-deploying-gitaly-cluster). +- Upgrade to the latest possible version of GitLab, to take advantage of improvements and bug fixes. + +To migrate to Gitaly Cluster: 1. Create the required storage. Refer to - [repository storage recommendations](faq.md#what-are-some-repository-storage-recommendations). + [repository storage recommendations](praefect.md#repository-storage-recommendations). 1. Create and configure [Gitaly Cluster](praefect.md). 1. [Move the repositories](../operations/moving_repositories.md#move-repositories). To migrate to Gitaly Cluster, existing repositories stored outside Gitaly Cluster must be moved. There is no automatic migration but the moves can be scheduled with the GitLab API. -WARNING: -Some [known database inconsistency issues](#known-issues) exist in Gitaly Cluster. We recommend you -remain on your current service for now. - -NOTE: -GitLab requires a `default` repository storage to be configured. +Even if you don't use the `default` repository storage, you must ensure it is configured. [Read more about this limitation](configure_gitaly.md#gitlab-requires-a-default-repository-storage). ## Migrate off Gitaly Cluster diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md index 17f94f912ee..a0a2e43569c 100644 --- a/doc/administration/gitaly/monitoring.md +++ b/doc/administration/gitaly/monitoring.md @@ -155,6 +155,7 @@ The following metrics are available from the `/metrics` endpoint: - `gitaly_praefect_node_latency_bucket`, a histogram measuring the latency in Gitaly returning health check information to Praefect. This indicates Praefect connection saturation. Available in GitLab 12.10 and later. +- `gitaly_praefect_connections_total`, the total number of connections to Praefect. [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4220) in GitLab 14.7. To monitor [strong consistency](index.md#strong-consistency), you can use the following Prometheus metrics: diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index fb9f3e9c361..f6166d713b1 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -19,9 +19,6 @@ Configure Gitaly Cluster using either: Smaller GitLab installations may need only [Gitaly itself](index.md). -To upgrade a Gitaly Cluster, follow the documentation for -[zero downtime upgrades](../../update/zero_downtime.md#gitaly-cluster). - ## Requirements The minimum recommended configuration for a Gitaly Cluster requires: @@ -54,7 +51,7 @@ Achieving acceptable latency between Gitaly nodes: are designed for this type of synchronization. Latency of less than 2ms should be sufficient for Gitaly Cluster. If you can't provide low network latencies for replication (for example, between distant locations), consider Geo. For -more information, see [How does Gitaly Cluster compare to Geo](faq.md#how-does-gitaly-cluster-compare-to-geo). +more information, see [Comparison to Geo](index.md#comparison-to-geo). Gitaly Cluster [components](index.md#components) communicate with each other over many routes. Your firewall rules must allow the following for Gitaly Cluster to function properly: @@ -64,6 +61,7 @@ allow the following for Gitaly Cluster to function properly: | GitLab | Praefect load balancer | `2305` | `3305` | | Praefect load balancer | Praefect | `2305` | `3305` | | Praefect | Gitaly | `8075` | `9999` | +| Praefect | GitLab (internal API) | `80` | `443` | | Gitaly | GitLab (internal API) | `80` | `443` | | Gitaly | Praefect load balancer | `2305` | `3305` | | Gitaly | Praefect | `2305` | `3305` | @@ -74,6 +72,16 @@ Gitaly does not directly connect to Praefect. However, requests from Gitaly to t load balancer may still be blocked unless firewalls on the Praefect nodes allow traffic from the Gitaly nodes. +### Praefect database storage + +The requirements are relatively low because the database contains only metadata of: + +- Where repositories are located. +- Some queued work. + +It depends on the number of repositories, but a useful minimum is 5-10 GB, similar to the main +GitLab application database. + ## Setup Instructions If you [installed](https://about.gitlab.com/install/) GitLab using the Omnibus GitLab package @@ -168,6 +176,18 @@ The following options are available: - Use a cloud-managed PostgreSQL service. AWS [Relational Database Service](https://aws.amazon.com/rds/) is recommended. +Setting up PostgreSQL creates empty Praefect tables. For more information, see the +[relevant troubleshooting section](troubleshooting.md#relation-does-not-exist-errors). + +#### Running GitLab and Praefect databases on the same server + +The GitLab application database and the Praefect database can be run on the same server. However, Praefect should have +its own database server when using Omnibus GitLab PostgreSQL. If there is a failover, Praefect isn't aware and starts to +fail as the database it's trying to use would either: + +- Be unavailable. +- In read-only mode. + #### Manual database setup To complete this section you need: @@ -278,8 +298,12 @@ reads distribution caching is enabled by configuration #### Use PgBouncer To reduce PostgreSQL resource consumption, we recommend setting up and configuring -[PgBouncer](https://www.pgbouncer.org/) in front of the PostgreSQL instance. To do -this, you must point Praefect to PgBouncer by setting database parameters on Praefect configuration: +[PgBouncer](https://www.pgbouncer.org/) in front of the PostgreSQL instance. However, PgBouncer isn't required because +Praefect makes a low number of connections. If you choose to use PgBouncer, you can use the same PgBouncer instance for +both the GitLab application database and the Praefect database. + +To configure PgBouncer in front of the PostgreSQL instance, you must point Praefect to PgBouncer by setting database +parameters on Praefect configuration: ```ruby praefect['database_host'] = PGBOUNCER_HOST @@ -475,7 +499,7 @@ Updates to example must be made at: # Some metrics run queries against the database. Enabling separate database metrics allows # these metrics to be collected when the metrics are - # scraped on a separate /db_metrics endpoint. + # scraped on a separate /db_metrics endpoint. praefect['separate_database_metrics'] = true ``` @@ -519,7 +543,7 @@ Updates to example must be made at: WARNING: If you have data on an already existing storage called `default`, you should configure the virtual storage with another name and - [migrate the data to the Gitaly Cluster storage](index.md#migrating-to-gitaly-cluster) + [migrate the data to the Gitaly Cluster storage](index.md#migrate-to-gitaly-cluster) afterwards. Replace `PRAEFECT_INTERNAL_TOKEN` with a strong secret, which is used by @@ -632,12 +656,8 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - When running Praefect sub-commands such as `dial-nodes` and `list-untracked-repositories` from the command line with [Gitaly TLS enabled](configure_gitaly.md#enable-tls-support), you must set the `SSL_CERT_DIR` or `SSL_CERT_FILE` environment variable so that the Gitaly certificate is trusted. For example: @@ -651,6 +671,8 @@ Note the following: This allows you to do a gradual transition from unencrypted to encrypted traffic, if necessary. + To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + To configure Praefect with TLS: **For Omnibus GitLab** @@ -992,7 +1014,7 @@ Particular attention should be shown to: WARNING: If you have existing data stored on the default Gitaly storage, - you should [migrate the data your Gitaly Cluster storage](index.md#migrating-to-gitaly-cluster) + you should [migrate the data your Gitaly Cluster storage](index.md#migrate-to-gitaly-cluster) first. ```ruby @@ -1107,7 +1129,7 @@ Particular attention should be shown to: #### Use TCP for existing GitLab instances When adding Gitaly Cluster to an existing Gitaly instance, the existing Gitaly storage -must use a TCP address. If `gitaly_address` is not specified, then a Unix socket is used, +must be listening on TCP/TLS. If `gitaly_address` is not specified, then a Unix socket is used, which prevents the communication with the cluster. For example: @@ -1116,7 +1138,7 @@ For example: git_data_dirs({ 'default' => { 'gitaly_address' => 'tcp://old-gitaly.internal:8075' }, 'cluster' => { - 'gitaly_address' => 'tcp://<PRAEFECT_LOADBALANCER_HOST>:2305', + 'gitaly_address' => 'tls://<PRAEFECT_LOADBALANCER_HOST>:3305', 'gitaly_token' => '<praefect_external_token>' } }) @@ -1223,6 +1245,18 @@ You can configure: If `default_replication_factor` is unset, the repositories are always replicated on every node defined in `virtual_storages`. If a new node is introduced to the virtual storage, both new and existing repositories are replicated to the node automatically. +### Repository storage recommendations + +The size of the required storage can vary between instances and depends on the set +[replication factor](index.md#replication-factor). You might want to include implementing +repository storage redundancy. + +For a replication factor: + +- Of `1`: NFS, Gitaly, and Gitaly Cluster have roughly the same storage requirements. +- More than `1`: The amount of required storage is `used space * replication factor`. `used space` + should include any planned future growth. + ## Repository verification > [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4080) in GitLab 15.0. diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 6de2acf1792..ecd68234b01 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -359,7 +359,7 @@ Add the `-older-than` option to avoid showing repositories that are the process Praefect database. Replace `<duration>` with a time duration (for example, `5s`, `10m`, or `1h`). Defaults to `6h`. ```shell -sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml list-untracked-repositories -older-than <duration> +sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml list-untracked-repositories -older-than <duration> ``` Only repositories with a creation time before the specified duration are considered. diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index c79ed1d1707..a1f542ddac8 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -8,9 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w Refer to the information below when troubleshooting Gitaly and Gitaly Cluster. -Before troubleshooting, see the Gitaly and Gitaly Cluster -[frequently asked questions](faq.md). - ## Troubleshoot Gitaly The following sections provide possible solutions to Gitaly errors. @@ -349,6 +346,14 @@ that do not exist in a repository. "error":"not found: .gitlab-ci.yml" ``` +### Git pushes are slow when Dynatrace is enabled + +Dynatrace can cause the `/opt/gitlab/embedded/bin/gitaly-hooks` reference transaction hook, +to take several seconds to start up and shut down. `gitaly-hooks` is executed twice when users +push, which causes a significant delay. + +If Git pushes are too slow when Dynatrace is enabled, disable Dynatrace. + ## Troubleshoot Praefect (Gitaly Cluster) The following sections provide possible solutions to Gitaly Cluster errors. diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index 71b8439f70a..93409b54c0d 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/inactive_project_deletion.md b/doc/administration/inactive_project_deletion.md index a2d2093c57b..224b52d420e 100644 --- a/doc/administration/inactive_project_deletion.md +++ b/doc/administration/inactive_project_deletion.md @@ -9,32 +9,35 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0 [with a flag](../administration/feature_flags.md) named `inactive_projects_deletion`. Disabled by default. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `inactive_projects_deletion`. On GitLab.com, this feature is not available. This feature is not ready for production use. Administrators of large GitLab instances can find that over time, projects become inactive and are no longer used. -These projects take up unnecessary disk space. With inactive project deletion, you can identify these projects, warn -the maintainers ahead of time, and then delete the projects if they remain inactive. When an inactive project is +These projects take up unnecessary disk space. With inactive project deletion, you can identify these projects, warn +the maintainers ahead of time, and then delete the projects if they remain inactive. When an inactive project is deleted, the action generates an audit event that it was performed by the first active administrator. ## Configure inactive project deletion -You can configure inactive projects deletion or turn it off using the -[Application settings API](../api/settings.md#change-application-settings). +You can configure inactive projects deletion or turn it off using either: + +- [The GitLab API](#using-the-api) (GitLab 15.0 and later). +- [The GitLab UI](#using-the-gitlab-ui) (GitLab 15.1 and later). The following options are available: -- `delete_inactive_projects`: Enable or disable inactive project deletion. -- `inactive_projects_min_size_mb`: Minimum size (MB) of inactive projects to be considered for deletion. - Projects smaller in size than this threshold aren't considered inactive. -- `inactive_projects_delete_after_months`: Minimum duration (months) after which a project is scheduled for deletion if - it continues be inactive. -- `inactive_projects_send_warning_email_after_months`: Minimum duration (months) after which a deletion warning email is - sent if a project continues to be inactive. The warning email is sent to users with the Owner and Maintainer roles of - the inactive project. This duration should be less than the `inactive_projects_delete_after_months` duration. +- **Delete inactive projects** (`delete_inactive_projects`): Enable or disable inactive project deletion. +- **Delete inactive projects that exceed** (`inactive_projects_min_size_mb`): Minimum size (MB) of inactive projects to + be considered for deletion. Projects smaller in size than this threshold aren't considered inactive. +- **Delete project after** (`inactive_projects_delete_after_months`): Minimum duration (months) after which a project is + scheduled for deletion if it continues be inactive. +- **Send warning email** (`inactive_projects_send_warning_email_after_months`): Minimum duration (months) after which a + deletion warning email is sent if a project continues to be inactive. The warning email is sent to users with the + Owner and Maintainer roles of the inactive project. This duration must be less than the + **Delete project after** (`inactive_projects_delete_after_months`) duration. -For example: +For example (using the API): - `delete_inactive_projects` enabled. - `inactive_projects_min_size_mb` set to `50`. @@ -49,6 +52,22 @@ In this scenario, when a project's size is: with the scheduled date of deletion. - More than 12 months, the project is scheduled for deletion. +### Using the API + +You can use the [Application settings API](../api/settings.md#change-application-settings) to configure inactive projects. + +### Using the GitLab UI + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85575) in GitLab 15.1. + +To configure inactive projects with the GitLab UI: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Repository**. +1. Expand **Repository maintenance**. +1. In the **Inactive project deletion** section, configure the necessary options. +1. Select **Save changes**. + ## Determine when a project was last active You can view a project's activities and determine when the project was last active in the following ways: diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 2fd57036169..95301ec026a 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -68,7 +68,8 @@ this method only supports replies, and not the other features of [incoming email ## Accepted headers -> Accepting `Received` headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81489) in GitLab 14.9 [with a flag](feature_flags.md) named `use_received_header_for_incoming_emails`. Enabled by default. +> - Accepting `Received` headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81489) in GitLab 14.9 [with a flag](feature_flags.md) named `use_received_header_for_incoming_emails`. Enabled by default. +> - Accepting `Received` headers: [feature flag](feature_flags.md) named `use_received_header_for_incoming_emails` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/362596) in GitLab 14.1. Email is processed correctly when a configured email address is present in one of the following headers (sorted in the order they are checked): @@ -132,7 +133,7 @@ issue by email" or "[Create new merge request by email](../user/project/merge_requests/creating_merge_requests.md#by-sending-an-email)" features by using a project's unique address as the email when signing up for Slack. This would send a confirmation email, which would create a new issue or -merge request on the project owned by the attacker, allowing them to click the +merge request on the project owned by the attacker, allowing them to select the confirmation link and validate their account on your company's private Slack instance. diff --git a/doc/administration/index.md b/doc/administration/index.md index 1d8dcd34d68..29fb65e2deb 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" description: 'Learn how to install, configure, update, and maintain your GitLab instance.' @@ -65,7 +65,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. requests diffs external storage. - [Broadcast Messages](../user/admin_area/broadcast_messages.md): Send messages to GitLab users through the UI. -- [Elasticsearch](../integration/elasticsearch.md): Enable Elasticsearch to +- [Elasticsearch](../integration/advanced_search/elasticsearch.md): Enable Elasticsearch to empower Advanced Search. Use when you deal with a huge amount of data. - [External Classification Policy Authorization](../user/admin_area/settings/external_authorization.md) - [Add a license](../user/admin_area/license.md): Add a license to unlock @@ -120,7 +120,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Creating users](../user/profile/account/create_accounts.md): Create users manually or through authentication integrations. - [Libravatar](libravatar.md): Use Libravatar instead of Gravatar for user avatars. -- [Sign-up restrictions](../user/admin_area/settings/sign_up_restrictions.md): block email addresses of specific domains, or whitelist only specific domains. +- [Sign-up restrictions](../user/admin_area/settings/sign_up_restrictions.md): block email addresses of specific domains, or allow only specific domains. - [Access restrictions](../user/admin_area/settings/visibility_and_access_controls.md#configure-enabled-git-access-protocols): Define which Git access protocols can be used to talk to GitLab (SSH, HTTP, HTTPS). - [Authentication and Authorization](auth/index.md): Configure external authentication with LDAP, SAML, CAS, and additional providers. - [Sync LDAP](auth/ldap/index.md) @@ -194,7 +194,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Monitoring GitLab](monitoring/index.md): - [Monitoring uptime](../user/admin_area/monitoring/health_check.md): Check the server status using the health check endpoint. - - [IP whitelist](monitoring/ip_whitelist.md): Monitor endpoints that provide health check information when probed. + - [IP allowlist](monitoring/ip_whitelist.md): Monitor endpoints that provide health check information when probed. - [Monitoring GitHub imports](monitoring/github_imports.md): The GitLab GitHub Importer displays Prometheus metrics to monitor the health and progress of the importer. ### Performance Monitoring @@ -239,7 +239,7 @@ who are aware of the risks. - Related links: - [GitLab Developer Documentation](../development/index.md) - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html) - - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/ch-testing-with-openssl.html) + - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/testing-with-openssl/index.html) - [`strace` zine](https://wizardzines.com/zines/strace/) - GitLab.com-specific resources: - [Group SAML/SCIM setup](troubleshooting/group_saml_scim.md) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 8d3fec3b19e..d86414ae285 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -133,8 +133,9 @@ Limit the maximum daily member invitations allowed per group hierarchy. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61151) in GitLab 13.12. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/330133) in GitLab 14.1. +> - [Limit changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89591) from per-hook to per-top-level namespace in GitLab 15.1. -Limit the number of times any given webhook can be called per minute. +Limit the number of times a webhook can be called per minute, per top-level namespace. This only applies to project and group webhooks. Calls over the rate limit are logged into `auth.log`. @@ -164,6 +165,14 @@ This setting limits global search requests. | Authenticated user | 30 | | Unauthenticated user | 10 | +### Pipeline creation rate limit + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362475) in GitLab 15.0. + +This setting limits the request rate to the pipeline creation endpoints. + +Read more about [pipeline creation rate limits](../user/admin_area/settings/rate_limit_on_pipelines_creation.md). + ## Gitaly concurrency limit Clone traffic can put a large strain on your Gitaly service. To prevent such workloads from overwhelming your Gitaly server, you can set concurrency limits in Gitaly's configuration file. @@ -383,6 +392,38 @@ Plan.default.actual_limits.update!(ci_active_jobs: 500) Set the limit to `0` to disable it. +### Number of pipelines running concurrently + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32823) in GitLab 12.5. + +The total number of pipelines running concurrently can be limited per project. +When enabled, the limit is checked each time a new pipeline is created. +Without a concurrent pipelines limit, a sudden flood of triggered pipelines could +overwhelm the instance resources. + +If a new pipeline would cause the total number of pipelines to exceed the limit, +the pipeline fails with a `The pipeline activity limit was exceeded.` error. + +On [GitLab Premium](https://about.gitlab.com/pricing/) self-managed or +higher installations, this limit is defined under a `default` plan that affects all +projects. This limit is disabled (`0`) by default. GitLab SaaS subscribers have different +limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect +all projects under that plan. + +To set this limit for a self-managed installation, enable the **Maximum number of active pipelines per project** +[setting in the Admin Area](../user/admin_area/settings/continuous_integration.md#set-cicd-limits). + +Alternatively, you can run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): + +```ruby +# If limits don't exist for the default plan, you can create one with: +# Plan.default.create_limits! + +Plan.default.actual_limits.update!(ci_active_pipelines: 100) +``` + +Set the limit to `0` to disable it. + ### Maximum time jobs can run > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16777) in GitLab 12.3. @@ -514,6 +555,26 @@ Plan.default.actual_limits.update!(ci_daily_pipeline_schedule_triggers: 1440) This limit is [enabled on GitLab.com](../user/gitlab_com/index.md#gitlab-cicd). +### Limit the number of schedule rules defined for security policy project + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335659) in GitLab 15.1. + +You can limit the total number of schedule rules per security policy project. This limit is +checked each time policies with schedule rules are updated. If a new schedule rule would +cause the total number of schedule rules to exceed the limit, the new schedule rule is +not processed. + +By default, self-managed instances do not limit the number of processable schedule rules. + +To set this limit for a self-managed installation, run the following in the +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): + +```ruby +Plan.default.actual_limits.update!(security_policy_scan_execution_schedules: 100) +``` + +This limit is [enabled on GitLab.com](../user/gitlab_com/index.md#gitlab-cicd). + ### Number of instance level variables > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216097) in GitLab 13.1. @@ -646,7 +707,7 @@ Update `ci_jobs_trace_size_limit` with the new value in megabytes: Plan.default.actual_limits.update!(ci_jobs_trace_size_limit: 125) ``` -GitLab Runner also has an +GitLab Runner also has an [`output_limit` setting](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) that configures the maximum log size in a runner. Jobs that exceed the runner limit continue to run, but the log is truncated when it hits the limit. @@ -825,9 +886,9 @@ prevent any more changes from rendering. For more information about these limits Reports that go over the 20 MB limit aren't loaded. Affected reports: -- [Merge request security reports](../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports) +- [Merge request security reports](../ci/testing/index.md#security-reports) - [CI/CD parameter `artifacts:expose_as`](../ci/yaml/index.md#artifactsexpose_as) -- [Unit test reports](../ci/unit_test_reports.md) +- [Unit test reports](../ci/testing/unit_test_reports.md) ## Advanced Search limits @@ -864,7 +925,7 @@ indexed, which have a separate limit. For more information, read - For self-managed installations, the field length is unlimited by default. You can configure this limit for self-managed installations when you -[enable Elasticsearch](../integration/elasticsearch.md#enable-advanced-search). +[enable Elasticsearch](../integration/advanced_search/elasticsearch.md#enable-advanced-search). Set the limit to `0` to disable it. ## Wiki limits @@ -981,7 +1042,7 @@ varies by file type: If a branch is merged while open merge requests still point to it, GitLab can retarget merge requests pointing to the now-merged branch. To learn more, read -[Branch retargeting on merge](../user/project/merge_requests/getting_started.md#branch-retargeting-on-merge). +[Update merge requests when target branch merges](../user/project/merge_requests/index.md#update-merge-requests-when-target-branch-merges). ## CDN-based limits on GitLab.com @@ -999,3 +1060,11 @@ The [secure files API](../api/secure_files.md) enforces the following limits: - Files must be smaller than 5 MB. - Projects cannot have more than 100 secure files. + +## Changelog API limits + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default. + +The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits: + +- The commit range between `from` and `to` cannot exceed 15000 commits. diff --git a/doc/administration/integration/kroki.md b/doc/administration/integration/kroki.md index 0f02e3783a5..a6fcfe6c80f 100644 --- a/doc/administration/integration/kroki.md +++ b/doc/administration/integration/kroki.md @@ -1,6 +1,6 @@ --- -stage: Ecosystem -group: Integrations +stage: Plan +group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/integration/mailgun.md b/doc/administration/integration/mailgun.md index 5a56aed4427..fbad201be7d 100644 --- a/doc/administration/integration/mailgun.md +++ b/doc/administration/integration/mailgun.md @@ -1,6 +1,6 @@ --- -stage: Growth -group: Expansion +stage: Plan +group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference, howto --- @@ -9,26 +9,37 @@ type: reference, howto When you use Mailgun to send emails for your GitLab instance and [Mailgun](https://www.mailgun.com/) integration is enabled and configured in GitLab, you can receive their webhook for -permanent invite email failures. To set up the integration, you must: +tracking delivery failures. To set up the integration, you must: 1. [Configure your Mailgun domain](#configure-your-mailgun-domain). 1. [Enable Mailgun integration](#enable-mailgun-integration). -After completing the integration, Mailgun `permanent_failure` webhooks are sent to your GitLab instance. +After completing the integration, Mailgun `temporary_failure` and `permanent_failure` webhooks are sent to your GitLab instance. ## Configure your Mailgun domain -Before you can enable Mailgun in GitLab, set up your own Mailgun permanent failure endpoint to receive the webhooks. +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the `/-/members/mailgun/permanent_failures` URL in GitLab 15.0. +> [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the URL to handle both temporary and permanent failures in GitLab 15.0. + +Before you can enable Mailgun in GitLab, set up your own Mailgun endpoints to receive the webhooks. Using the [Mailgun webhook guide](https://www.mailgun.com/blog/a-guide-to-using-mailguns-webhooks/): 1. Add a webhook with the **Event type** set to **Permanent Failure**. -1. Fill in the URL of your instance and include the `/-/members/mailgun/permanent_failures` path. - - Example: `https://myinstance.gitlab.com/-/members/mailgun/permanent_failures` +1. Enter the URL of your instance and include the `/-/mailgun/webhooks` path. + + For example: + + ```plaintext + https://myinstance.gitlab.com/-/mailgun/webhooks + ``` + +1. Add another webhook with the **Event type** set to **Temporary Failure**. +1. Enter the URL of your instance and use the same `/-/mailgun/webhooks` path. ## Enable Mailgun integration -After configuring your Mailgun domain for the permanent failures endpoint, +After configuring your Mailgun domain for the webhook endpoints, you're ready to enable the Mailgun integration: 1. Sign in to GitLab as an [Administrator](../../user/permissions.md) user. diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 0d75880bdd1..ebf76cc6aec 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -24,7 +24,7 @@ There are various configuration options to help GitLab server administrators: - Enabling/disabling Git LFS support. - Changing the location of LFS object storage. -- Setting up object storage supported by [Fog](http://fog.io/about/provider_documentation.html). +- Setting up object storage supported by [Fog](https://fog.io/about/provider_documentation.html). ### Configuration for Omnibus installations @@ -57,7 +57,7 @@ In `config/gitlab.yml`: You can store LFS objects in remote object storage. This allows you to reduce reads and writes to the local disk, and free up disk space significantly. -GitLab is tightly integrated with `Fog`, so you can refer to its [documentation](http://fog.io/about/provider_documentation.html) +GitLab is tightly integrated with `Fog`, so you can refer to its [documentation](https://fog.io/about/provider_documentation.html) to check which storage services can be integrated with GitLab. You can also use external object storage in a private local network. For example, [MinIO](https://min.io/) is a standalone object storage service that works with GitLab instances. diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index 142cf2f3685..442c233a536 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: howto diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md index 45fd6e65078..e6a1c0c25d5 100644 --- a/doc/administration/load_balancer.md +++ b/doc/administration/load_balancer.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/logs.md b/doc/administration/logs.md index c89c485e7d3..5a2c1190896 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -61,7 +61,7 @@ Some of these services have their own environment variables to override the log | GitLab Export | `INFO` | `EXPORT_DEBUG` | | GitLab Geo | `INFO` | | | GitLab Import | `INFO` | `IMPORT_DEBUG` | -| GitLab QA Runtime | `ERROR` | `QA_DEBUG` | +| GitLab QA Runtime | `INFO` | `QA_LOG_LEVEL` | | Google APIs | `INFO` | | | Rack Timeout | `ERROR` | | | Sidekiq (server) | `INFO` | | @@ -144,7 +144,8 @@ Line breaks were added to examples for legibility: "db_duration_s":0.08, "view_duration_s":2.39, "duration_s":20.54, - "pid": 81836 + "pid": 81836, + "worker_id":"puma_0" } ``` @@ -167,7 +168,8 @@ seconds: - `redis_<instance>_duration_s`: Total time to retrieve data from a Redis instance - `redis_<instance>_read_bytes`: Total bytes read from a Redis instance - `redis_<instance>_write_bytes`: Total bytes written to a Redis instance -- `pid`: Process ID of the Puma worker +- `pid`: The worker's Linux process ID (changes when workers restart) +- `worker_id`: The worker's logical ID (does not change when workers restart) User clone and fetch activity using HTTP transport appears in the log as `action: git_upload_pack`. @@ -240,6 +242,7 @@ Starting with GitLab 12.5, if an error occurs, an "view_duration_s":2.39, "duration_s":20.54, "pid": 81836, + "worker_id": "puma_0", "exception.class": "NameError", "exception.message": "undefined local variable or method `adsf' for #<Admin::DashboardController:0x00007ff3c9648588>", "exception.backtrace": [ @@ -319,7 +322,10 @@ It helps you see requests made directly to the API. For example: "username":"root", "queue_duration":100.31, "gitaly_calls":30, - "gitaly_duration":5.36 + "gitaly_duration":5.36, + "pid": 81836, + "worker_id": "puma_0", + ... } ``` @@ -546,6 +552,7 @@ Sidekiq. For example: "created_at":"2018-04-03T22:57:21.930Z", "enqueued_at":"2018-04-03T22:57:21.931Z", "pid":10077, + "worker_id":"sidekiq_0", "message":"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", "job_status":"done", "duration":0.139, diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index 41eed500594..9128f978dd5 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -14,7 +14,7 @@ Once Maintenance Mode is enabled, in-progress actions finish relatively quickly In that state, various maintenance tasks are easier, and services can be stopped completely or be further degraded for a much shorter period of time than might otherwise be needed. For example, stopping cron jobs and draining queues should be fairly quick. -Maintenance Mode allows most external actions that do not change internal state. On a high-level, HTTP POST, PUT, PATCH, and DELETE requests are blocked and a detailed overview of [how special cases are handled](#rest-api) is available. +Maintenance Mode allows most external actions that do not change internal state. On a high-level, HTTP `POST`, `PUT`, `PATCH`, and `DELETE` requests are blocked and a detailed overview of [how special cases are handled](#rest-api) is available. ## Enable Maintenance Mode @@ -107,22 +107,22 @@ Notification emails continue to arrive, but emails that require database writes, ### REST API -For most JSON requests, POST, PUT, PATCH, and DELETE are blocked, and the API returns a 403 response with the error message: `You cannot perform write operations on a read-only instance`. Only the following requests are allowed: +For most JSON requests, `POST`, `PUT`, `PATCH`, and `DELETE` are blocked, and the API returns a 403 response with the error message: `You cannot perform write operations on a read-only instance`. Only the following requests are allowed: |HTTP request | Allowed routes | Notes | |:----:|:--------------------------------------:|:----:| -| POST | `/admin/application_settings/general` | To allow updating application settings in the administrator UI | -| PUT | `/api/v4/application/settings` | To allow updating application settings with the API | -| POST | `/users/sign_in` | To allow users to log in. | -| POST | `/users/sign_out`| To allow users to log out. | -| POST | `/oauth/token` | To allow users to log in to a Geo secondary for the first time. | -| POST | `/admin/session`, `/admin/session/destroy` | To allow [Admin Mode for GitLab administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158) | -| POST | Paths ending with `/compare`| Git revision routes. | -| POST | `.git/git-upload-pack` | To allow Git pull/clone. | -| POST | `/api/v4/internal` | [internal API routes](../../development/internal_api/index.md) | -| POST | `/admin/sidekiq` | To allow management of background jobs in the Admin Area | -| POST | `/admin/geo` | To allow updating Geo Nodes in the administrator UI | -| POST | `/api/v4/geo_replication`| To allow certain Geo-specific administrator UI actions on secondary sites | +| `POST` | `/admin/application_settings/general` | To allow updating application settings in the administrator UI | +| `PUT` | `/api/v4/application/settings` | To allow updating application settings with the API | +| `POST` | `/users/sign_in` | To allow users to log in. | +| `POST` | `/users/sign_out`| To allow users to log out. | +| `POST` | `/oauth/token` | To allow users to log in to a Geo secondary for the first time. | +| `POST` | `/admin/session`, `/admin/session/destroy` | To allow [Admin Mode for GitLab administrators](https://gitlab.com/groups/gitlab-org/-/epics/2158) | +| `POST` | Paths ending with `/compare`| Git revision routes. | +| `POST` | `.git/git-upload-pack` | To allow Git pull/clone. | +| `POST` | `/api/v4/internal` | [internal API routes](../../development/internal_api/index.md) | +| `POST` | `/admin/sidekiq` | To allow management of background jobs in the Admin Area | +| `POST` | `/admin/geo` | To allow updating Geo Nodes in the administrator UI | +| `POST` | `/api/v4/geo_replication`| To allow certain Geo-specific administrator UI actions on secondary sites | ### GraphQL API diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 6f2e3cce0fa..2553638291d 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -54,7 +54,7 @@ you create the project again, it's created in its default state. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, go to **Settings > Metrics and profiling** and expand **Self monitoring**. 1. Toggle **Self monitoring** off. -1. In the confirmation dialog that opens, click **Delete self monitoring project**. +1. In the confirmation dialog that opens, select **Delete self monitoring project**. It can take a few seconds for it to be deleted. 1. After the project is deleted, GitLab displays a message confirming your action. @@ -82,10 +82,10 @@ you [configure it manually](../../../user/project/integrations/prometheus.md#man ## Take action on Prometheus alerts **(ULTIMATE)** -You can [add a webhook](../../../operations/metrics/alerts.md#external-prometheus-instances) -to the Prometheus configuration for GitLab to receive notifications of any alerts. +You can [add a Prometheus integration](../../../operations/incident_management/integrations.md) +to GitLab to receive notifications of any alerts. -Once the webhook is setup, you can +Once the integration is setup, you can [take action on incoming alerts](../../../operations/metrics/alerts.md#trigger-actions-from-alerts). ## Add custom metrics to the self monitoring project diff --git a/doc/administration/monitoring/index.md b/doc/administration/monitoring/index.md index 381c807dbc5..82a6da1d56a 100644 --- a/doc/administration/monitoring/index.md +++ b/doc/administration/monitoring/index.md @@ -20,7 +20,7 @@ Explore our features to monitor your GitLab instance: importer with various Prometheus metrics. - [Monitoring uptime](../../user/admin_area/monitoring/health_check.md): Check the server status using the health check endpoint. - - [IP whitelists](ip_whitelist.md): Configure GitLab for monitoring endpoints that + - [IP allowlists](ip_whitelist.md): Configure GitLab for monitoring endpoints that provide health check information when probed. - [`nginx_status`](https://docs.gitlab.com/omnibus/settings/nginx.html#enablingdisabling-nginx_status): Monitor your NGINX server status. diff --git a/doc/administration/monitoring/ip_allowlist.md b/doc/administration/monitoring/ip_allowlist.md new file mode 100644 index 00000000000..adf9516733a --- /dev/null +++ b/doc/administration/monitoring/ip_allowlist.md @@ -0,0 +1,57 @@ +--- +stage: Data Stores +group: Memory +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# IP whitelist **(FREE SELF)** + +NOTE: +We intend to [rename IP whitelist as `IP allowlist`](https://gitlab.com/groups/gitlab-org/-/epics/3478). + +GitLab provides some [monitoring endpoints](../../user/admin_area/monitoring/health_check.md) +that provide health check information when probed. + +To control access to those endpoints via IP whitelisting, you can add single +hosts or use IP ranges: + +**For Omnibus installations** + +1. Open `/etc/gitlab/gitlab.rb` and add or uncomment the following: + + ```ruby + gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] + ``` + +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. + +--- + +**For installations using cloud native Helm charts** + +You can set the required IPs under the `gitlab.webservice.monitoring.ipWhitelist` key. For example: + +```yaml +gitlab: + webservice: + monitoring: + # Monitoring IP whitelist + ipWhitelist: + - 0.0.0.0/0 # Default +``` + +--- + +**For installations from source** + +1. Edit `config/gitlab.yml`: + + ```yaml + monitoring: + # by default only local IPs are allowed to access monitoring resources + ip_whitelist: + - 127.0.0.0/8 + - 192.168.0.1 + ``` + +1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/monitoring/ip_whitelist.md b/doc/administration/monitoring/ip_whitelist.md index b8347ba3f0d..9fb4ffe3089 100644 --- a/doc/administration/monitoring/ip_whitelist.md +++ b/doc/administration/monitoring/ip_whitelist.md @@ -1,57 +1,11 @@ --- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'ip_allowlist.md' +remove_date: '2022-08-31' --- -# IP whitelist **(FREE SELF)** +This document was moved to [another location](ip_allowlist.md). -NOTE: -We intend to [rename IP whitelist as `IP allowlist`](https://gitlab.com/groups/gitlab-org/-/epics/3478). - -GitLab provides some [monitoring endpoints](../../user/admin_area/monitoring/health_check.md) -that provide health check information when probed. - -To control access to those endpoints via IP whitelisting, you can add single -hosts or use IP ranges: - -**For Omnibus installations** - -1. Open `/etc/gitlab/gitlab.rb` and add or uncomment the following: - - ```ruby - gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] - ``` - -1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. - ---- - -**For installations using cloud native Helm charts** - -You can set the required IPs under the `gitlab.webservice.monitoring.ipWhitelist` key. For example: - -```yaml -gitlab: - webservice: - monitoring: - # Monitoring IP whitelist - ipWhitelist: - - 0.0.0.0/0 # Default -``` - ---- - -**For installations from source** - -1. Edit `config/gitlab.yml`: - - ```yaml - monitoring: - # by default only local IPs are allowed to access monitoring resources - ip_whitelist: - - 127.0.0.0/8 - - 192.168.0.1 - ``` - -1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. +<!-- This redirect file can be deleted after <2022-08-31>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/monitoring/performance/performance_bar.md b/doc/administration/monitoring/performance/performance_bar.md index bc311f73dfe..aefd7a49b8b 100644 --- a/doc/administration/monitoring/performance/performance_bar.md +++ b/doc/administration/monitoring/performance/performance_bar.md @@ -21,7 +21,7 @@ From left to right, the performance bar displays: - **Current Host**: the current host serving the page. - **Database queries**: the time taken (in milliseconds) and the total number - of database queries, displayed in the format `00ms / 00 (00 cached) pg`. Click to display + of database queries, displayed in the format `00ms / 00 (00 cached) pg`. Select to display a modal window with more details. You can use this to see the following details for each query: - **In a transaction**: shows up below the query if it was executed in @@ -35,21 +35,21 @@ From left to right, the performance bar displays: GitLab features. The name shown is the same name used to configure database connections in GitLab. - **Gitaly calls**: the time taken (in milliseconds) and the total number of - [Gitaly](../../gitaly/index.md) calls. Click to display a modal window with more + [Gitaly](../../gitaly/index.md) calls. Select to display a modal window with more details. - **Rugged calls**: the time taken (in milliseconds) and the total number of [Rugged](../../nfs.md#improving-nfs-performance-with-gitlab) calls. - Click to display a modal window with more details. + Select to display a modal window with more details. - **Redis calls**: the time taken (in milliseconds) and the total number of - Redis calls. Click to display a modal window with more details. + Redis calls. Select to display a modal window with more details. - **Elasticsearch calls**: the time taken (in milliseconds) and the total number of - Elasticsearch calls. Click to display a modal window with more details. + Elasticsearch calls. Select to display a modal window with more details. - **External HTTP calls**: the time taken (in milliseconds) and the total - number of external calls to other systems. Click to display a modal window + number of external calls to other systems. Select to display a modal window with more details. - **Load timings** of the page: if your browser supports load timings, several values in milliseconds, separated by slashes. - Click to display a modal window with more details. The values, from left to right: + Select to display a modal window with more details. The values, from left to right: - **Backend**: time needed for the base page to load. - [**First Contentful Paint**](https://web.dev/first-contentful-paint/): Time until something was visible to the user. Displays `NaN` if your browser does not @@ -112,7 +112,7 @@ for a given group: 1. On the left sidebar, select **Settings > Metrics and profiling** (`admin/application_settings/metrics_and_profiling`), and expand **Profiling - Performance bar**. -1. Click **Allow non-administrators access to the performance bar**. +1. Select **Allow non-administrators access to the performance bar**. 1. In the **Allow access to members of the following group** field, provide the full path of the group allowed to access the performance. -1. Click **Save changes**. +1. Select **Save changes**. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 22f7419be9a..51360800d66 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -137,8 +137,8 @@ The steps below are the minimum necessary to configure a Monitoring node running # Enable service discovery for Prometheus consul['enable'] = true - consul['monitoring_service_discovery'] = true - consul['configuration'] = { + consul['monitoring_service_discovery'] = true + consul['configuration'] = { retry_join: %w(10.0.0.1 10.0.0.2 10.0.0.3), # The addresses can be IPs or FQDNs } @@ -205,7 +205,7 @@ To use an external Prometheus server: ``` 1. Install and set up a dedicated Prometheus instance, if necessary, using the [official installation instructions](https://prometheus.io/docs/prometheus/latest/installation/). -1. Add the Prometheus server IP address to the [monitoring IP whitelist](../ip_whitelist.md). For example: +1. Add the Prometheus server IP address to the [monitoring IP allowlist](../ip_whitelist.md). For example: ```ruby gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.0.1'] @@ -381,11 +381,12 @@ memory, disk, and CPU utilization. [Read more about the node exporter](node_exporter.md). -### Puma exporter +### Web exporter -The Puma exporter allows you to measure various Puma metrics. +The web exporter is a dedicated metrics server that allows splitting end-user and Prometheus traffic +into two separate applications to improve performance and availability. -[Read more about the Puma exporter](puma_exporter.md). +[Read more about the web exporter](puma_exporter.md). ### Redis exporter diff --git a/doc/administration/monitoring/prometheus/puma_exporter.md b/doc/administration/monitoring/prometheus/puma_exporter.md index 794e2c10b25..a3e095f5f09 100644 --- a/doc/administration/monitoring/prometheus/puma_exporter.md +++ b/doc/administration/monitoring/prometheus/puma_exporter.md @@ -1,29 +1,11 @@ --- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'web_exporter.md' +remove_date: '2022-09-01' --- -# Puma exporter **(FREE SELF)** +This document was moved to [another location](web_exporter.md). -You can use the [Puma exporter](https://github.com/sapcc/puma-exporter) -to measure various Puma metrics. - -To enable the Puma exporter: - -1. [Enable Prometheus](index.md#configuring-prometheus). -1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines. Make sure - `puma['exporter_enabled']` is set to `true`: - - ```ruby - puma['exporter_enabled'] = true - puma['exporter_address'] = "127.0.0.1" - puma['exporter_port'] = 8083 - ``` - -1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - -Prometheus begins collecting performance data from the Puma exporter exposed at `localhost:8083`. - -For more information on using Puma with GitLab, see [Puma](../../operations/puma.md). +<!-- This redirect file can be deleted after <2022-09-01>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/administration/monitoring/prometheus/web_exporter.md b/doc/administration/monitoring/prometheus/web_exporter.md new file mode 100644 index 00000000000..4b449a1d74e --- /dev/null +++ b/doc/administration/monitoring/prometheus/web_exporter.md @@ -0,0 +1,53 @@ +--- +stage: Data Stores +group: Memory +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Web exporter (dedicated metrics server) **(FREE SELF)** + +When [monitoring GitLab with Prometheus](index.md), GitLab runs various collectors that +sample the application for data related to usage, load and performance. GitLab can then make +this data available to a Prometheus scraper by running one or more Prometheus exporters. +A Prometheus exporter is an HTTP server that serializes metric data into a format the +Prometheus scraper understands. + +NOTE: +This page is about web application metrics. +To export background job metrics, learn how to [configure the Sidekiq metrics server](../../sidekiq.md#configure-the-sidekiq-metrics-server). + +We provide two mechanisms by which web application metrics can be exported: + +- Through the main Rails application. This means [Puma](../../operations/puma.md), the application server we use, + makes metric data available via its own `/-/metrics` endpoint. This is the default, + and is described in [GitLab Metrics](index.md#gitlab-metrics). We recommend this + default for small GitLab installations where the amount of metrics collected is small. +- Through a dedicated metrics server. Enabling this server will cause Puma to launch an + additional process whose sole responsibility is to serve metrics. This approach leads + to better fault isolation and performance for very large GitLab installations, but + comes with additional memory use. We recommend this approach for medium to large + GitLab installations that seek high performance and availability. + +Both the dedicated server and the Rails `/-/metrics` endpoint serve the same data, so +they are functionally equivalent and differ merely in their performance characteristics. + +To enable the dedicated server: + +1. [Enable Prometheus](index.md#configuring-prometheus). +1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines. Make sure + `puma['exporter_enabled']` is set to `true`: + + ```ruby + puma['exporter_enabled'] = true + puma['exporter_address'] = "127.0.0.1" + puma['exporter_port'] = 8083 + ``` + +1. When using the GitLab-bundled Prometheus, make sure that its `scrape_config` is pointing + to `localhost:8083/metrics`. Refer to the [Adding custom scrape configurations](index.md#adding-custom-scrape-configurations) page + for how to configure scraper targets. For external Prometheus setup, refer to + [Using an external Prometheus server](index.md#using-an-external-prometheus-server) instead. +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +Metrics can now be served and scraped from `localhost:8083/metrics`. diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index b0c50d63e78..d9866a6c09f 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 0560a8813df..7e416ed560a 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -239,18 +239,19 @@ in the `connection` setting. The connection settings match those provided by [fog-aws](https://github.com/fog/fog-aws): -| Setting | Description | Default | -|---------------------------------|------------------------------------|---------| -| `provider` | Always `AWS` for compatible hosts. | `AWS` | -| `aws_access_key_id` | AWS credentials, or compatible. | | -| `aws_secret_access_key` | AWS credentials, or compatible. | | -| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | -| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | -| `region` | AWS region. | | -| `host` | S3 compatible host for when not using AWS. For example, `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | -| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false`. | -| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys. | `false` | +| Setting | Description | Default | +|---------------------------------------------|------------------------------------|---------| +| `provider` | Always `AWS` for compatible hosts. | `AWS` | +| `aws_access_key_id` | AWS credentials, or compatible. | | +| `aws_secret_access_key` | AWS credentials, or compatible. | | +| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | +| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | +| `region` | AWS region. | | +| `host` | S3 compatible host for when not using AWS. For example, `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | +| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | +| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false`. | +| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys. | `false` | +| `aws_credentials_refresh_threshold_seconds` | Sets the [automatic refresh threshold](https://github.com/fog/fog-aws#controlling-credential-refresh-time-with-iam-authentication) when using temporary credentials in IAM. | `15` | #### Oracle Cloud S3 connection settings @@ -540,6 +541,10 @@ supported by consolidated configuration form, refer to the following guides: | [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | | [Pages content](pages/index.md#using-object-storage) | **{check-circle}** Yes | +WARNING: +The use of [encrypted S3 buckets](#encrypted-s3-buckets) with non-consolidated configuration is not supported. +You may start getting [ETag mismatch errors](#etag-mismatch) if you use it. + ### Other alternatives to file system storage If you're working to [scale out](reference_architectures/index.md) your GitLab implementation, @@ -765,7 +770,7 @@ Prerequisites: 1. [Install](https://rclone.org/downloads/) Rclone. 1. Configure Rclone by running the following: - + ```shell rclone config ``` @@ -778,7 +783,7 @@ Prerequisites: rclone ls old:uploads | head ``` - This should print a partial list of the objects currently stored in your `uploads` bucket. If you get an error, or if + This should print a partial list of the objects currently stored in your `uploads` bucket. If you get an error, or if the list is empty, go back and update your Rclone configuration using `rclone config`. 1. Perform an initial copy. You do not need to take your GitLab server offline for this step. @@ -788,7 +793,7 @@ Prerequisites: ``` 1. After the first sync completes, use the web UI or command-line interface of your new object storage provider to - verify that there are objects in the new bucket. If there are none, or if you encounter an error while running `rclone + verify that there are objects in the new bucket. If there are none, or if you encounter an error while running `rclone sync`, check your Rclone configuration and try again. After you have done at least one successful Rclone copy from the old location to the new location, schedule maintenance and take your GitLab server offline. During your maintenance window you must do two things: diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 75f2ad5ed26..cffe682a80b 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md index cd3a53b7c63..2a14fa1d312 100644 --- a/doc/administration/operations/extra_sidekiq_routing.md +++ b/doc/administration/operations/extra_sidekiq_routing.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -136,15 +136,15 @@ GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee `queue_selector` supports the following operators, listed from highest to lowest precedence: -- `|` - the logical OR operator. For example, `query_a|query_b` (where `query_a` +- `|` - the logical `OR` operator. For example, `query_a|query_b` (where `query_a` and `query_b` are queries made up of the other operators here) will include queues that match either query. -- `&` - the logical AND operator. For example, `query_a&query_b` (where +- `&` - the logical `AND` operator. For example, `query_a&query_b` (where `query_a` and `query_b` are queries made up of the other operators here) will only include queues that match both queries. -- `!=` - the NOT IN operator. For example, `feature_category!=issue_tracking` +- `!=` - the `NOT IN` operator. For example, `feature_category!=issue_tracking` excludes all queues from the `issue_tracking` feature category. -- `=` - the IN operator. For example, `resource_boundary=cpu` includes all +- `=` - the `IN` operator. For example, `resource_boundary=cpu` includes all queues that are CPU bound. - `,` - the concatenate set operator. For example, `feature_category=continuous_integration,pages` includes all queues from @@ -152,8 +152,8 @@ to lowest precedence: example is also possible using the OR operator, but allows greater brevity, as well as being lower precedence. -The operator precedence for this syntax is fixed: it's not possible to make AND -have higher precedence than OR. +The operator precedence for this syntax is fixed: it's not possible to make `AND` +have higher precedence than `OR`. [In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and later, as with the standard queue group syntax above, a single `*` as the diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index a0ad2e24a4c..9403634d905 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 7ccfa2739bb..fe531407eb2 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/operations/moving_repositories.md b/doc/administration/operations/moving_repositories.md index f0eb5792a96..1459b707e5c 100644 --- a/doc/administration/operations/moving_repositories.md +++ b/doc/administration/operations/moving_repositories.md @@ -27,7 +27,7 @@ For more information, see: querying and scheduling snippet repository moves. - [The API documentation](../../api/group_repository_storage_moves.md) details the endpoints for querying and scheduling group repository moves **(PREMIUM SELF)**. -- [Migrating to Gitaly Cluster](../gitaly/index.md#migrating-to-gitaly-cluster). +- [Migrate to Gitaly Cluster](../gitaly/index.md#migrate-to-gitaly-cluster). ### Move Repositories @@ -276,7 +276,7 @@ If the source or target directory has many contents, this startup phase of `rsyn server. You can reduce the workload of `rsync` by dividing its work into smaller pieces, and sync one repository at a time. -In addition to `rsync` we use [GNU Parallel](http://www.gnu.org/software/parallel/). +In addition to `rsync` we use [GNU Parallel](https://www.gnu.org/software/parallel/). This utility is not included in GitLab, so you must install it yourself with `apt` or `yum`. @@ -377,3 +377,14 @@ sudo -u git -H bundle exec rake gitlab:list_repos SINCE='2015-10-1 12:00 UTC' |\ /home/git/repositories \ /mnt/gitlab/repositories ``` + +## Troubleshooting + +See the following for information on troubleshooting repository moves. + +### Repository move fails for archived projects + +Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/363670), +[archived projects](../../user/project/settings/index.md#advanced-settings) fail to move even though the data is cloned +by Gitaly. Make sure archived projects are +[unarchived](../../user/project/settings/index.md#unarchiving-a-project) before initiating a move. diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index 12a8b2faadc..0025ca109e5 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -252,19 +252,19 @@ To switch from Unicorn to Puma: | `unicorn['enable']` | `puma['enable']` | | `unicorn['worker_timeout']` | `puma['worker_timeout']` | | `unicorn['worker_processes']` | `puma['worker_processes']` | - | n/a | `puma['ha']` | - | n/a | `puma['min_threads']` | - | n/a | `puma['max_threads']` | + | Not applicable | `puma['ha']` | + | Not applicable | `puma['min_threads']` | + | Not applicable | `puma['max_threads']` | | `unicorn['listen']` | `puma['listen']` | | `unicorn['port']` | `puma['port']` | | `unicorn['socket']` | `puma['socket']` | | `unicorn['pidfile']` | `puma['pidfile']` | - | `unicorn['tcp_nopush']` | n/a | - | `unicorn['backlog_socket']` | n/a | + | `unicorn['tcp_nopush']` | Not applicable | + | `unicorn['backlog_socket']` | Not applicable | | `unicorn['somaxconn']` | `puma['somaxconn']` | - | n/a | `puma['state_path']` | + | Not applicable | `puma['state_path']` | | `unicorn['log_directory']` | `puma['log_directory']` | - | `unicorn['worker_memory_limit_min']` | n/a | + | `unicorn['worker_memory_limit_min']` | Not applicable | | `unicorn['worker_memory_limit_max']` | `puma['per_worker_max_memory_mb']` | | `unicorn['exporter_enabled']` | `puma['exporter_enabled']` | | `unicorn['exporter_address']` | `puma['exporter_address']` | @@ -281,4 +281,4 @@ To switch from Unicorn to Puma: ## Related topics -- [Use the Puma exporter to measure various Puma metrics](../monitoring/prometheus/puma_exporter.md) +- [Use a dedicated metrics server to export web metrics](../monitoring/prometheus/puma_exporter.md) diff --git a/doc/administration/operations/rails_console.md b/doc/administration/operations/rails_console.md index a93365c08b2..df039ee734f 100644 --- a/doc/administration/operations/rails_console.md +++ b/doc/administration/operations/rails_console.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index e48ac6e65eb..d9558c3d7a6 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Memory info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index 4fcc5d7c916..3141312612e 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -53,7 +53,7 @@ the GitLab server itself, but your setup may vary. If the CA is only used for GitLab consider putting this in the `Match User git` section (described below). -The SSH certificates being issued by that CA **MUST** have a "key ID" +The SSH certificates being issued by that CA **must** have a "key ID" corresponding to that user's username on GitLab, for example (some output omitted for brevity): diff --git a/doc/administration/package_information/defaults.md b/doc/administration/package_information/defaults.md index 54e68392aa5..65cb8ef1e6c 100644 --- a/doc/administration/package_information/defaults.md +++ b/doc/administration/package_information/defaults.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -31,7 +31,7 @@ by default: | GitLab Exporter | Yes | Port | X | 9168 | | Sidekiq exporter | Yes | Port | X | 8082 | | Sidekiq health check | No | Port | X | 8092[^Sidekiq-health] | -| Puma exporter | No | Port | X | 8083 | +| Web exporter | No | Port | X | 8083 | | Geo PostgreSQL | No | Socket | Port (5431) | X | | Redis Sentinel | No | Port | X | 26379 | | Incoming email | No | Port | X | 143 | diff --git a/doc/administration/package_information/deprecation_policy.md b/doc/administration/package_information/deprecation_policy.md index 7298bce6c95..420690866e5 100644 --- a/doc/administration/package_information/deprecation_policy.md +++ b/doc/administration/package_information/deprecation_policy.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/package_information/index.md b/doc/administration/package_information/index.md index c908f526569..746961e8613 100644 --- a/doc/administration/package_information/index.md +++ b/doc/administration/package_information/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -14,11 +14,11 @@ at [bundling dependencies document](omnibus_packages.md). The released package versions are in the format `MAJOR.MINOR.PATCH-EDITION.OMNIBUS_RELEASE` -| Component | Meaning | Example | -|-------------------|---------|---------| -| MAJOR.MINOR.PATCH | The GitLab version this corresponds to. | 13.3.0 | -| EDITION | The edition of GitLab this corresponds to. | ee | -| OMNIBUS_RELEASE | The Omnibus GitLab release. Usually, this is 0. This is incremented if we need to build a new package without changing the GitLab version. | 0 | +| Component | Meaning | Example | +|---------------------|---------|---------| +| `MAJOR.MINOR.PATCH` | The GitLab version this corresponds to. | 13.3.0 | +| `EDITION` | The edition of GitLab this corresponds to. | ee | +| `OMNIBUS_RELEASE` | The Omnibus GitLab release. Usually, this is 0. This is incremented if we need to build a new package without changing the GitLab version. | 0 | ## Licenses @@ -87,7 +87,7 @@ Depending on the init system, this `WARNING` can be one of: /sbin/init: unrecognized option '--version' ``` -when the underlying init system *IS NOT* upstart. +when the underlying init system *is not* upstart. ```plaintext -.mount loaded active mounted / diff --git a/doc/administration/package_information/licensing.md b/doc/administration/package_information/licensing.md index d27c1df0ccf..e5afe6b85dd 100644 --- a/doc/administration/package_information/licensing.md +++ b/doc/administration/package_information/licensing.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/package_information/omnibus_packages.md b/doc/administration/package_information/omnibus_packages.md index 21e3f5711ba..d19f5f67b74 100644 --- a/doc/administration/package_information/omnibus_packages.md +++ b/doc/administration/package_information/omnibus_packages.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/package_information/postgresql_versions.md b/doc/administration/package_information/postgresql_versions.md index 1b3e49cfb82..8aab4121a0b 100644 --- a/doc/administration/package_information/postgresql_versions.md +++ b/doc/administration/package_information/postgresql_versions.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -21,6 +21,9 @@ For example: [Find out which versions of PostgreSQL (and other components) ship with each Omnibus GitLab release](https://gitlab-org.gitlab.io/omnibus-gitlab/licenses.html). +The lowest supported PostgreSQL versions are listed in the +[installation requirements](../../install/requirements.md#postgresql-requirements). + Read more about update policies and warnings in the PostgreSQL [upgrade docs](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server). diff --git a/doc/administration/package_information/signed_packages.md b/doc/administration/package_information/signed_packages.md index 24857dcfc27..351c2ac5d5f 100644 --- a/doc/administration/package_information/signed_packages.md +++ b/doc/administration/package_information/signed_packages.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/package_information/supported_os.md b/doc/administration/package_information/supported_os.md index 647620858d9..f2838cd779b 100644 --- a/doc/administration/package_information/supported_os.md +++ b/doc/administration/package_information/supported_os.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -15,21 +15,23 @@ page](https://about.gitlab.com/install/). The following lists the currently supported OSs and their possible EOL dates. -| OS Version | First supported GitLab version | Arch | OS EOL | Details | -| ---------------- | ------------------------------ | --------------- | ------------- | ------------------------------------------------------------ | -| AlmaLinux 8 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | 2029 | <https://almalinux.org/> | -| CentOS 7 | GitLab CE / GitLab EE 7.10.0 | x86_64 | June 2024 | <https://wiki.centos.org/About/Product> | -| CentOS 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, aarch64 | Dec 2021 | <https://wiki.centos.org/About/Product> | -| Debian 9 | GitLab CE / GitLab EE 9.3.0 | amd64 | 2022 | <https://wiki.debian.org/LTS> | -| Debian 10 | GitLab CE / GitLab EE 12.2.0 | amd64, arm64 | 2024 | <https://wiki.debian.org/LTS> | -| Debian 11 | GitLab CE / GitLab EE 14.6.0 | amd64, arm64 | 2026 | <https://wiki.debian.org/LTS> | -| OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | Nov 2022 | <https://en.opensuse.org/Lifetime> | -| RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | May 2024 | <https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates> | -| SLES 12 | GitLab EE 9.0.0 | x86_64 | Oct 2027 | <https://www.suse.com/lifecycle/> | -| Ubuntu 18.04 | GitLab CE / GitLab EE 10.7.0 | amd64 | April 2023 | <https://wiki.ubuntu.com/Releases> | -| Ubuntu 20.04 | GitLab CE / GitLab EE 13.2.0 | amd64, arm64 | April 2025 | <https://wiki.ubuntu.com/Releases> | -| Amazon Linux 2 | GitLab CE / GitLab EE 14.9.0 | amd64, arm64 | June 2023 | <https://aws.amazon.com/amazon-linux-2/faqs/> | -| Raspberry Pi OS (Buster) (formerly known as Raspbian Buster) | GitLab CE 12.2.0 | armhf | 2024 | <https://www.raspberrypi.com/news/new-old-functionality-with-raspberry-pi-os-legacy/> | +| OS Version | First supported GitLab version | Arch | Install Docs | OS EOL | Details | +| ------------------------------------------------------------ | ------------------------------ | --------------- | :----------------------------------------------------------: | ---------- | ------------------------------------------------------------ | +| AlmaLinux 8 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [AlmaLinux Install Docs](https://about.gitlab.com/install/#almalinux-8) | 2029 | <https://almalinux.org/> | +| CentOS 7 | GitLab CE / GitLab EE 7.10.0 | x86_64 | [CentOS Install Docs](https://about.gitlab.com/install/#centos-7) | June 2024 | <https://wiki.centos.org/About/Product> | +| Debian 9 | GitLab CE / GitLab EE 9.3.0 | amd64 | [Debian Install Docs](https://about.gitlab.com/install/#debian) | 2022 | <https://wiki.debian.org/LTS> | +| Debian 10 | GitLab CE / GitLab EE 12.2.0 | amd64, arm64 | [Debian Install Docs](https://about.gitlab.com/install/#debian) | 2024 | <https://wiki.debian.org/LTS> | +| Debian 11 | GitLab CE / GitLab EE 14.6.0 | amd64, arm64 | [Debian Install Docs](https://about.gitlab.com/install/#debian) | 2026 | <https://wiki.debian.org/LTS> | +| OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [OpenSUSE Install Docs](https://about.gitlab.com/install/#opensuse-leap-15-3) | Nov 2022 | <https://en.opensuse.org/Lifetime> | +| RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | [Use CentOS Install Docs](https://about.gitlab.com/install/#centos-7) | May 2024 | [RHEL Details](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates) | +| SLES 12 | GitLab EE 9.0.0 | x86_64 | [Use OpenSUSE Install Docs](https://about.gitlab.com/install/#opensuse-leap-15-3) | Oct 2027 | <https://www.suse.com/lifecycle/> | + +| Oracle Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Docs](https://about.gitlab.com/install/#centos-7) | Jul 2024 | <https://www.oracle.com/a/ocom/docs/elsp-lifetime-069338.pdf> | +| Scientific Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Docs](https://about.gitlab.com/install/#centos-7) | June 2024 | <https://scientificlinux.org/downloads/sl-versions/sl7/> | +| Ubuntu 18.04 | GitLab CE / GitLab EE 10.7.0 | amd64 | [Ubuntu Install Docs](https://about.gitlab.com/install/#ubuntu) | April 2023 | <https://wiki.ubuntu.com/Releases> | +| Ubuntu 20.04 | GitLab CE / GitLab EE 13.2.0 | amd64, arm64 | [Ubuntu Install Docs](https://about.gitlab.com/install/#ubuntu) | April 2025 | <https://wiki.ubuntu.com/Releases> | +| Amazon Linux 2 | GitLab CE / GitLab EE 14.9.0 | amd64, arm64 | [Amazon Linux 2 Instal Docsl](https://about.gitlab.com/install/#amazonlinux-2) | June 2023 | <https://aws.amazon.com/amazon-linux-2/faqs/> | +| Raspberry Pi OS (Buster) (formerly known as Raspbian Buster) | GitLab CE 12.2.0 | armhf | [Raspberry Pi Install Docs](https://about.gitlab.com/install/#raspberry-pi-os) | 2024 | [Raspberry Pi Details](https://www.raspberrypi.com/news/new-old-functionality-with-raspberry-pi-os-legacy/) | NOTE: CentOS 8 was EOL on December 31, 2021. In GitLab 14.5 and later, @@ -82,6 +84,7 @@ release for them can be found below: | Raspbian Stretch | [June 2020](https://downloads.raspberrypi.org/raspbian/images/raspbian-2019-04-09/) | [GitLab CE](https://packages.gitlab.com/app/gitlab/raspberry-pi2/search?q=gitlab-ce_13.3&dist=raspbian%2Fstretch) 13.3 | | Debian Jessie | [June 2020](https://www.debian.org/News/2020/20200709) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_13.2&dist=debian%2Fjessie) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_13.2&dist=debian%2Fjessie) 13.3 | | CentOS 6 | [November 2020](https://wiki.centos.org/About/Product) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=13.6&filter=all&filter=all&dist=el%2F6) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=13.6&filter=all&filter=all&dist=el%2F6) 13.6 | +| CentOS 8 | [December 2021](https://wiki.centos.org/About/Product) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=14.6&filter=all&filter=all&dist=el%2F8) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=14.6&filter=all&filter=all&dist=el%2F8) 14.6 | | OpenSUSE 15.1 | [November 2020](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-13.12&dist=opensuse%2F15.1) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-13.12&dist=opensuse%2F15.1) 13.12 | | Ubuntu 16.04 | [April 2021](https://ubuntu.com/info/release-end-of-life) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_13.12&dist=ubuntu%2Fxenial) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_13.12&dist=ubuntu%2Fxenial) 13.12 | | OpenSUSE 15.2 | [December 2021](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-14.7&dist=opensuse%2F15.2) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-14.7&dist=opensuse%2F15.2) 14.7 | diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 0edad83cc18..2fa3eea54bd 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -317,6 +317,17 @@ the Container Registry by themselves, follow the steps below. 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. +### Increase token duration + +In GitLab, tokens for the Container Registry expire every five minutes. +To increase the token duration: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Container Registry**. +1. For the **Authorization token duration (minutes)**, update the value. +1. Select **Save changes**. + ## Configure storage for the Container Registry NOTE: @@ -477,7 +488,7 @@ To configure the `s3` storage driver in Omnibus: **Installations from source** -Configuring the storage driver is done in the registry configuration YML file created +Configuring the storage driver is done in the registry configuration YAML file created when you [deployed your Docker registry](https://docs.docker.com/registry/deploying/). `s3` storage driver example: @@ -600,7 +611,7 @@ However, this behavior is undesirable for registries used by internal hosts that **Installations from source** -1. Add the `redirect` flag to your registry configuration YML file: +1. Add the `redirect` flag to your registry configuration YAML file: ```yaml storage: @@ -652,7 +663,7 @@ For Omnibus GitLab installations: For installations from source: -1. Edit your registry configuration YML file: +1. Edit your registry configuration YAML file: ```yaml storage: @@ -705,7 +716,7 @@ In the examples below we set the Registry's port to `5010`. ## Disable Container Registry per project If Registry is enabled in your GitLab instance, but you don't need it for your -project, you can [disable it from your project's settings](../../user/project/settings/index.md#sharing-and-permissions). +project, you can [disable it from your project's settings](../../user/project/settings/index.md#configure-project-visibility-features-and-permissions). ## Use an external container registry with GitLab as an auth endpoint @@ -845,7 +856,7 @@ To configure a notification endpoint in Omnibus: **Installations from source** -Configuring the notification endpoint is done in your registry configuration YML file created +Configuring the notification endpoint is done in your registry configuration YAML file created when you [deployed your Docker registry](https://docs.docker.com/registry/deploying/). Example: @@ -1140,7 +1151,7 @@ for GitLab to run separately from Registry: - `gitlab_rails['registry_enabled']`, must be set to `true`. This setting signals to GitLab that it should allow Registry API requests. - `gitlab_rails['registry_api_url']`, default [set programmatically](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/10-3-stable/files/gitlab-cookbooks/gitlab/libraries/registry.rb#L52). This is the Registry URL used internally that users do not need to interact with, `registry['registry_http_addr']` with scheme. -- `gitlab_rails['registry_host']`, eg. `registry.gitlab.example`. Registry endpoint without the scheme, the address that gets shown to the end user. +- `gitlab_rails['registry_host']`, for example, `registry.gitlab.example`. Registry endpoint without the scheme, the address that gets shown to the end user. - `gitlab_rails['registry_port']`. Registry endpoint port, visible to the end user. - `gitlab_rails['registry_issuer']` must match the issuer in the Registry configuration. - `gitlab_rails['registry_key_path']`, path to the key that matches the certificate on the @@ -1159,7 +1170,7 @@ The flow described by the diagram above: 1. A user runs `docker login registry.gitlab.example` on their client. This reaches the web server (or LB) on port 443. 1. Web server connects to the Registry backend pool (by default, using port 5000). Since the user - didn’t provide a valid token, the Registry returns a 401 HTTP code and the URL (`token_realm` from + didn't provide a valid token, the Registry returns a 401 HTTP code and the URL (`token_realm` from Registry configuration) where to get one. This points to the GitLab API. 1. The Docker client then connects to the GitLab API and obtains a token. 1. The API signs the token with the registry key and hands it to the Docker client @@ -1169,7 +1180,7 @@ Reference: <https://docs.docker.com/registry/spec/auth/token/> ### Communication between GitLab and Registry -Registry doesn’t have a way to authenticate users internally so it relies on +Registry doesn't have a way to authenticate users internally so it relies on GitLab to validate credentials. The connection between Registry and GitLab is TLS encrypted. The key is used by GitLab to sign the tokens while the certificate is used by Registry to validate the signature. By default, a self-signed certificate key pair is generated @@ -1226,29 +1237,6 @@ mounting the Docker daemon and setting `privileged = false` in the GitLab Runner Additional information about this: [issue 18239](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/18239). -### `unauthorized: authentication required` when pushing large images - -Example error: - -```shell -docker push gitlab.example.com/myproject/docs:latest -The push refers to a repository [gitlab.example.com/myproject/docs] -630816f32edb: Preparing -530d5553aec8: Preparing -... -4b0bab9ff599: Waiting -d1c800db26c7: Waiting -42755cf4ee95: Waiting -unauthorized: authentication required -``` - -GitLab has a default token expiration of 5 minutes for the registry. When pushing -larger images, or images that take longer than 5 minutes to push, users may -encounter this error. On GitLab.com, the expiration time is 15 minutes. - -Administrators can increase the token duration in **Admin Area > Settings > -CI/CD > Container Registry > Authorization token duration (minutes)**. - ### Docker login attempt fails with: 'token signed by untrusted key' [Registry relies on GitLab to validate credentials](#architecture-of-gitlab-container-registry) @@ -1309,7 +1297,10 @@ openssl rsa -noout -modulus -in /var/opt/gitlab/gitlab-rails/etc/gitlab-registry ``` If the two pieces of the certificate do not align, remove the files and run `gitlab-ctl reconfigure` -to regenerate the pair. If you have overridden the automatically generated self-signed pair with +to regenerate the pair. The pair is recreated using the existing values in `/etc/gitlab/gitlab-secrets.json` if they exist. To generate a new pair, +delete the `registry` section in your `/etc/gitlab/gitlab-secrets.json` before running `gitlab-ctl reconfigure`. + +If you have overridden the automatically generated self-signed pair with your own certificates and have made sure that their contents align, you can delete the 'registry' section in your `/etc/gitlab/gitlab-secrets.json` and run `gitlab-ctl reconfigure`. @@ -1378,7 +1369,7 @@ You can add a configuration option for backwards compatibility. **For installations from source** -1. Edit the YML configuration file you created when you [deployed the registry](https://docs.docker.com/registry/deploying/). Add the following snippet: +1. Edit the YAML configuration file you created when you [deployed the registry](https://docs.docker.com/registry/deploying/). Add the following snippet: ```yaml compatibility: @@ -1426,7 +1417,7 @@ and a simple solution would be to enable relative URLs in the Registry. **For installations from source** -1. Edit the YML configuration file you created when you [deployed the registry](https://docs.docker.com/registry/deploying/). Add the following snippet: +1. Edit the YAML configuration file you created when you [deployed the registry](https://docs.docker.com/registry/deploying/). Add the following snippet: ```yaml http: @@ -1741,7 +1732,7 @@ In this case, follow these steps: ```ruby gitlab_rails['registry_enabled'] = true ``` - + 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Try the removal again. diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index ef00127a70e..56786f334b0 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -26,7 +26,7 @@ The Packages feature allows GitLab to act as a repository and supports the follo ## Accepting contributions -The below table lists formats that are not supported, but are accepting Community contributions for. Consider contributing to GitLab. This [development documentation](../../development/packages.md) +The below table lists formats that are not supported, but are accepting Community contributions for. Consider contributing to GitLab. This [development documentation](../../development/packages/index.md) guides you through the process. <!-- vale gitlab.Spelling = NO --> diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index f144c60fcfe..1535de0c2c2 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -83,7 +83,7 @@ added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/infrastructure/-/issue ### DNS configuration GitLab Pages expect to run on their own virtual host. In your DNS server/provider -add a [wildcard DNS A record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the +add a [wildcard DNS `A` record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the host that GitLab runs. For example, an entry would look like this: ```plaintext @@ -257,7 +257,6 @@ control over how the Pages daemon runs and serves content in your environment. | `auth_scope` | The OAuth application scope to use for authentication. Must match GitLab Pages OAuth application settings. Leave blank to use `api` scope by default. | | `gitlab_server` | Server to use for authentication when access control is enabled; defaults to GitLab `external_url`. | | `headers` | Specify any additional http headers that should be sent to the client with each response. Multiple headers can be given as an array, header and value as one string, for example `['my-header: myvalue', 'my-other-header: my-other-value']` | -| `inplace_chroot` | [REMOVED in GitLab 14.3.](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/561) On [systems that don't support bind-mounts](index.md#gitlab-pages-fails-to-start-in-docker-container), this instructs GitLab Pages to `chroot` into its `pages_path` directory. Some caveats exist when using in-place `chroot`; refer to the GitLab Pages [README](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md#caveats) for more information. | | `enable_disk` | Allows the GitLab Pages daemon to serve content from disk. Shall be disabled if shared disk storage isn't available. | | `insecure_ciphers` | Use default list of cipher suites, may contain insecure ones like 3DES and RC4. | | `internal_gitlab_server` | Internal GitLab server address used exclusively for API requests. Useful if you want to send that traffic over an internal load balancer. Defaults to GitLab `external_url`. | @@ -681,35 +680,44 @@ Follow the steps below to configure the proxy listener of GitLab Pages. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -## Set global maximum pages size per project **(FREE SELF)** +## Set global maximum size of each GitLab Pages site **(FREE SELF)** + +Prerequisites: + +- Only GitLab administrators can edit this setting. To set the global maximum pages size for a project: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Preferences**. 1. Expand **Pages**. -1. Edit the **Maximum size of pages**. +1. Enter a value under **Maximum size of pages**. 1. Select **Save changes**. -## Override maximum pages size per project or group **(PREMIUM SELF)** +## Set maximum size of each GitLab Pages site in a group **(PREMIUM SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16610) in GitLab 12.7. - -NOTE: -Only GitLab administrators are able to view and override the **Maximum size of Pages** setting. +Prerequisites: -To override the global maximum pages size for a specific project: +- You must have at least the Maintainer role for the group. -1. On the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Pages**. -1. Enter a value under **Maximum size of pages** in MB. -1. Select **Save changes**. - -To override the global maximum pages size for a specific group: +To set the maximum size of each GitLab Pages site in a group, overriding the inherited setting: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Settings > General**. 1. Expand **Pages**. +1. Enter a value under **Maximum size** in MB. +1. Select **Save changes**. + +## Set maximum size of GitLab Pages site in a project **(PREMIUM SELF)** + +Prerequisites: + +- You must have at least the Maintainer role for the project. + +To set the maximum size of GitLab Pages site in a project, overriding the inherited setting: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Pages**. 1. Enter a value under **Maximum size of pages** in MB. 1. Select **Save changes**. @@ -1187,7 +1195,8 @@ Rate limits are enforced using the following: Include any troubleshooting steps that you can foresee. If you know beforehand what issues one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. +important to describe those, too. Think of things that may go wrong and include them in +the section below. This is important to minimize requests for support, and to avoid doc comments with questions that you know someone might ask. @@ -1207,72 +1216,6 @@ sudo gitlab-ctl tail gitlab-pages You can also find the log file in `/var/log/gitlab/gitlab-pages/current`. -### `open /etc/ssl/ca-bundle.pem: permission denied` - -WARNING: -This issue is fixed in GitLab 14.3 and above, try upgrading GitLab first. - -GitLab Pages runs inside a `chroot` jail, usually in a uniquely numbered directory like -`/tmp/gitlab-pages-*`. - -In the jail, a bundle of trusted certificates is -provided at `/etc/ssl/ca-bundle.pem`. It's -[copied there](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/51) -from `/opt/gitlab/embedded/ssl/certs/cacert.pem` -as part of starting up Pages. - -If the permissions on the source file are incorrect (they should be `0644`), then -the file inside the `chroot` jail is also wrong. - -Pages logs errors in `/var/log/gitlab/gitlab-pages/current` like: - -```plaintext -x509: failed to load system roots and no roots provided -open /etc/ssl/ca-bundle.pem: permission denied -``` - -The use of a `chroot` jail makes this error misleading, as it is not -referring to `/etc/ssl` on the root file system. - -The fix is to correct the source file permissions and restart Pages: - -```shell -sudo chmod 644 /opt/gitlab/embedded/ssl/certs/cacert.pem -sudo gitlab-ctl restart gitlab-pages -``` - -### `dial tcp: lookup gitlab.example.com` and `x509: certificate signed by unknown authority` - -WARNING: -This issue is fixed in GitLab 14.3 and above, try upgrading GitLab first. - -When setting both `inplace_chroot` and `access_control` to `true`, you might encounter errors like: - -```plaintext -dial tcp: lookup gitlab.example.com on [::1]:53: dial udp [::1]:53: connect: cannot assign requested address -``` - -Or: - -```plaintext -open /opt/gitlab/embedded/ssl/certs/cacert.pem: no such file or directory -x509: certificate signed by unknown authority -``` - -The reason for those errors is that the files `resolv.conf`, `/etc/hosts/`, `/etc/nsswitch.conf` and `ca-bundle.pem` are missing inside the `chroot`. -The fix is to copy these files inside the `chroot`: - -```shell -sudo mkdir -p /var/opt/gitlab/gitlab-rails/shared/pages/etc/ssl -sudo mkdir -p /var/opt/gitlab/gitlab-rails/shared/pages/opt/gitlab/embedded/ssl/certs/ - -sudo cp /etc/resolv.conf /var/opt/gitlab/gitlab-rails/shared/pages/etc/ -sudo cp /etc/hosts /var/opt/gitlab/gitlab-rails/shared/pages/etc/ -sudo cp /etc/nsswitch.conf /var/opt/gitlab/gitlab-rails/shared/pages/etc/ -sudo cp /opt/gitlab/embedded/ssl/certs/cacert.pem /var/opt/gitlab/gitlab-rails/shared/pages/opt/gitlab/embedded/ssl/certs/ -sudo cp /opt/gitlab/embedded/ssl/certs/cacert.pem /var/opt/gitlab/gitlab-rails/shared/pages/etc/ssl/ca-bundle.pem -``` - ### `unsupported protocol scheme \"\""` If you see the following error: @@ -1541,7 +1484,7 @@ In GitLab 14.0-14.2 you can temporarily enable legacy storage and configuration To do that: -1. Please describe the issue you're seeing in [here](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). +1. Please describe the issue you're seeing in the [migration feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). 1. Edit `/etc/gitlab/gitlab.rb`: @@ -1551,28 +1494,6 @@ To do that: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### GitLab Pages fails to start in Docker container - -WARNING: -This issue is fixed in GitLab 14.3 and above, try upgrading GitLab first. - -The GitLab Pages daemon doesn't have permissions to bind mounts when it runs -in a Docker container. To overcome this issue, you must change the `chroot` -behavior: - -1. Edit `/etc/gitlab/gitlab.rb`. -1. Set the `inplace_chroot` to `true` for GitLab Pages: - - ```ruby - gitlab_pages['inplace_chroot'] = true - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -NOTE: -`inplace_chroot` option might not work with the other features, such as [Pages Access Control](#access-control). -The [GitLab Pages README](https://gitlab.com/gitlab-org/gitlab-pages#caveats) has more information about caveats and workarounds. - ### GitLab Pages deploy job fails with error "is not a recognized provider" If the **pages** job succeeds but the **deploy** job gives the error "is not a recognized provider": diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 6c148387d7d..4bb9fc56b6f 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -68,7 +68,7 @@ Before proceeding with the Pages configuration, make sure that: ### DNS configuration GitLab Pages expect to run on their own virtual host. In your DNS server/provider -you need to add a [wildcard DNS A record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the +you need to add a [wildcard DNS `A` record](https://en.wikipedia.org/wiki/Wildcard_DNS_record) pointing to the host that GitLab runs. For example, an entry would look like this: ```plaintext diff --git a/doc/administration/polling.md b/doc/administration/polling.md index 8bd28824f83..5699bf78c04 100644 --- a/doc/administration/polling.md +++ b/doc/administration/polling.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/postgresql/database_load_balancing.md b/doc/administration/postgresql/database_load_balancing.md index 1324666c32b..b03abf9881c 100644 --- a/doc/administration/postgresql/database_load_balancing.md +++ b/doc/administration/postgresql/database_load_balancing.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -97,9 +97,9 @@ For example, on an environment that has PostgreSQL running on the hosts `host1.e Service discovery allows GitLab to automatically retrieve a list of PostgreSQL hosts to use. It periodically -checks a DNS A record, using the IPs returned by this record as the addresses +checks a DNS `A` record, using the IPs returned by this record as the addresses for the secondaries. For service discovery to work, all you need is a DNS server -and an A record containing the IP addresses of your secondaries. +and an `A` record containing the IP addresses of your secondaries. When using Omnibus GitLab the provided [Consul](../consul.md) service works as a DNS server and returns PostgreSQL addresses via the `postgresql-ha.service.consul` @@ -125,23 +125,23 @@ record. For example: |----------------------|---------------------------------------------------------------------------------------------------|-----------| | `nameserver` | The nameserver to use for looking up the DNS record. | localhost | | `record` | The record to look up. This option is required for service discovery to work. | | -| `record_type` | Optional record type to look up, this can be either A or SRV (GitLab 12.3 and later) | A | +| `record_type` | Optional record type to look up, this can be either `A` or `SRV` (GitLab 12.3 and later) | `A` | | `port` | The port of the nameserver. | 8600 | | `interval` | The minimum time in seconds between checking the DNS record. | 60 | | `disconnect_timeout` | The time in seconds after which an old connection is closed, after the list of hosts was updated. | 120 | | `use_tcp` | Lookup DNS resources using TCP instead of UDP | false | If `record_type` is set to `SRV`, then GitLab continues to use round-robin algorithm -and ignores the `weight` and `priority` in the record. Since SRV records usually +and ignores the `weight` and `priority` in the record. Since `SRV` records usually return hostnames instead of IPs, GitLab needs to look for the IPs of returned hostnames -in the additional section of the SRV response. If no IP is found for a hostname, GitLab -needs to query the configured `nameserver` for ANY record for each such hostname looking for A or AAAA +in the additional section of the `SRV` response. If no IP is found for a hostname, GitLab +needs to query the configured `nameserver` for `ANY` record for each such hostname looking for `A` or `AAAA` records, eventually dropping this hostname from rotation if it can't resolve its IP. -The `interval` value specifies the _minimum_ time between checks. If the A +The `interval` value specifies the _minimum_ time between checks. If the `A` record has a TTL greater than this value, then service discovery honors said -TTL. For example, if the TTL of the A record is 90 seconds, then service -discovery waits at least 90 seconds before checking the A record again. +TTL. For example, if the TTL of the `A` record is 90 seconds, then service +discovery waits at least 90 seconds before checking the `A` record again. When the list of hosts is updated, it might take a while for the old connections to be terminated. The `disconnect_timeout` setting can be used to enforce an diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md index f4e4c7f8bef..5d693793a92 100644 --- a/doc/administration/postgresql/external.md +++ b/doc/administration/postgresql/external.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/postgresql/index.md b/doc/administration/postgresql/index.md index 15425a6d9f2..8b811deee16 100644 --- a/doc/administration/postgresql/index.md +++ b/doc/administration/postgresql/index.md @@ -1,12 +1,12 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Configuring PostgreSQL for scaling **(FREE SELF)** -In this section, you'll be guided through configuring a PostgreSQL database to +In this section, you are guided through configuring a PostgreSQL database to be used with GitLab in one of our [reference architectures](../reference_architectures/index.md). There are essentially three setups to choose from. diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index ed3c662eba3..8ae2b6497f8 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -7,7 +7,7 @@ type: reference # Working with the bundled PgBouncer service **(PREMIUM SELF)** -[PgBouncer](http://www.pgbouncer.org/) is used to seamlessly migrate database +[PgBouncer](https://www.pgbouncer.org/) is used to seamlessly migrate database connections between servers in a failover scenario. Additionally, it can be used in a non-fault-tolerant setup to pool connections, speeding up response time while reducing resource usage. @@ -227,12 +227,12 @@ the database. Each of the listed services below use the following formula to def - `headroom` can be configured via `DB_POOL_HEADROOM` environment variable (default to `10`) To calculate the `default_pool_size`, multiply the number of instances of `puma`, `sidekiq` and `geo-logcursor` by the -number of connections each can consume as per listed above. The total will be the suggested `default_pool_size`. +number of connections each can consume as per listed above. The total is the suggested `default_pool_size`. If you are using more than one PgBouncer with an internal Load Balancer, you may be able to divide the `default_pool_size` by the number of instances to guarantee an evenly distributed load between them. -The `pgbouncer['max_client_conn']` is the hard-limit of connections PgBouncer can accept. It's unlikely you will need +The `pgbouncer['max_client_conn']` is the hard limit of connections PgBouncer can accept. It's unlikely you need to change this. If you are hitting that limit, you may want to consider adding additional PgBouncers with an internal Load Balancer. diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index 84122149cb8..e9b607ad5d4 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -42,7 +42,7 @@ collections "**Consul** x3" as consul #e76a9b card "Database" as database { collections "**PGBouncer x3**\n//Consul//" as pgbouncer #4EA7FF - + card "**PostgreSQL** //Primary//\n//Patroni//\n//PgBouncer//\n//Consul//" as postgres_primary #4EA7FF collections "**PostgreSQL** //Secondary// **x2**\n//Patroni//\n//PgBouncer//\n//Consul//" as postgres_secondary #4EA7FF diff --git a/doc/administration/postgresql/standalone.md b/doc/administration/postgresql/standalone.md index b21625acb56..5428e44ccc0 100644 --- a/doc/administration/postgresql/standalone.md +++ b/doc/administration/postgresql/standalone.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index ad4cfd11474..b14c1f64efe 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments remove_date: '2022-08-22' diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index fba151fefe1..d66ff9afb94 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/geo.md b/doc/administration/raketasks/geo.md index 6306e54bdc6..5c6c99d099b 100644 --- a/doc/administration/raketasks/geo.md +++ b/doc/administration/raketasks/geo.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -33,7 +33,7 @@ sudo -u git -H bundle exec rake geo:git:housekeeping:incremental_repack RAILS_EN ### Full Repack This is equivalent of running `git repack -d -A --pack-kept-objects` on a -_bare_ repository which will optionally, write a reachability bitmap index +_bare_ repository which optionally, writes a reachability bitmap index when this is enabled in GitLab. **Omnibus Installation** diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index 0cdfd1c28ff..30c8518c209 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -12,7 +12,7 @@ which becomes the owner of the project. You can resume an import with the same command. Bear in mind that the syntax is very specific. Remove any spaces within the argument block and -before/after the brackets. Also, some shells (for example, `zsh`) can interpret the open/close brackets +before/after the brackets. Also, some shells (for example, Zsh) can interpret the open/close brackets (`[]`) separately. You may need to either escape the brackets or use double quotes. ## Caveats diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 82ebdcaab42..f4bd6f255e5 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 13f8dfdccc2..f623d4e4526 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/smtp.md b/doc/administration/raketasks/smtp.md index c738f1dcd00..bc78d54aa1b 100644 --- a/doc/administration/raketasks/smtp.md +++ b/doc/administration/raketasks/smtp.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/storage.md b/doc/administration/raketasks/storage.md index 689ed5c5824..27b899dd1b1 100644 --- a/doc/administration/raketasks/storage.md +++ b/doc/administration/raketasks/storage.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/uploads/migrate.md b/doc/administration/raketasks/uploads/migrate.md index aec75f0b302..c73840cb9ff 100644 --- a/doc/administration/raketasks/uploads/migrate.md +++ b/doc/administration/raketasks/uploads/migrate.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index 7dc813de14e..bf6dc4fd776 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/read_only_gitlab.md b/doc/administration/read_only_gitlab.md index b7e8397dd95..a601ba4cd7a 100644 --- a/doc/administration/read_only_gitlab.md +++ b/doc/administration/read_only_gitlab.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/redis/index.md b/doc/administration/redis/index.md index 23d13491c75..3581b27a4c8 100644 --- a/doc/administration/redis/index.md +++ b/doc/administration/redis/index.md @@ -1,6 +1,6 @@ --- type: index -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index 086057d80b4..9b1a456835a 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -1,6 +1,6 @@ --- type: howto -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -12,8 +12,8 @@ This is the documentation for the Omnibus GitLab packages. For using your own non-bundled Redis, follow the [relevant documentation](replication_and_failover_external.md). NOTE: -In Redis lingo, primary is called master. In this document, primary is used -instead of master, except the settings where `master` is required. +In Redis lingo, `primary` is called `master`. In this document, `primary` is used +instead of `master`, except the settings where `master` is required. Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index c19e42a5f14..998455e5621 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -1,6 +1,6 @@ --- type: howto -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -218,7 +218,7 @@ The following steps should be performed in the GitLab application server which ideally should not have Redis or Sentinels in the same machine: 1. Edit `/home/git/gitlab/config/resque.yml` following the example in - [resque.yml.example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to + [`resque.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/resque.yml.example), and uncomment the Sentinel lines, pointing to the correct server credentials: ```yaml diff --git a/doc/administration/redis/standalone.md b/doc/administration/redis/standalone.md index 8a3b88780a1..6e0e1eb886e 100644 --- a/doc/administration/redis/standalone.md +++ b/doc/administration/redis/standalone.md @@ -1,6 +1,6 @@ --- type: howto -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/redis/troubleshooting.md b/doc/administration/redis/troubleshooting.md index f4aab9d7b7f..4426adc5d51 100644 --- a/doc/administration/redis/troubleshooting.md +++ b/doc/administration/redis/troubleshooting.md @@ -1,6 +1,6 @@ --- type: reference -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index f70912dbecb..99272cdd226 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -18,23 +18,23 @@ full list of reference architectures, see > - **Test requests per second (RPS) rates:** API: 200 RPS, Web: 20 RPS, Git (Pull): 20 RPS, Git (Push): 4 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/10k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|-----------------------------------------------------|-------------|-------------------------|------------------|--------------|-----------| -| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|-----------------------------------------------------|----------------|-------------------------|------------------|----------------|----------------| +| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | +| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> @@ -86,7 +86,7 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 - + redis_cache -[hidden]-> redis_persistent } @@ -1620,18 +1620,12 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. - +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. - + necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS pass-through. Refer to the load balancers documentation on how to configure this. @@ -2186,7 +2180,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -2202,10 +2196,6 @@ There are two ways of specifying object storage configuration in GitLab: Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. -GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared via NFS on any GitLab Rails and Sidekiq nodes. - -In GitLab 13.6 and later, it's also recommended to switch to [Incremental logging](../job_logs.md#incremental-logging-architecture), which uses Redis instead of disk space for temporary caching of job logs. This is required when no NFS node has been deployed. - For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides based on what features you intend to use: @@ -2235,15 +2225,21 @@ in the future. </a> </div> +## Enable incremental logging + +GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared through NFS on any GitLab Rails and Sidekiq nodes. + +While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. + ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2317,18 +2313,18 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|------------------------------------------|-------|-----------------------|------------------|--------------| -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| PostgreSQL<sup>1</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|------------------------------------------|----------------|-----------------------|------------------|----------------| +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| PostgreSQL<sup>1</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Gitaly<sup>5</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> @@ -2381,7 +2377,7 @@ card "Database" as database { card "redis" as redis { collections "**Redis Persistent** x3" as redis_persistent #FF6347 collections "**Redis Cache** x3" as redis_cache #FF6347 - + redis_cache -[hidden]-> redis_persistent } diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index 8d5afd732d9..c356c7224cb 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -62,7 +62,7 @@ monitor .[#7FFFD4,norank]--> redis @enduml ``` -The diagram above shows that while GitLab can be installed on a single server, it is internally composed of multiple services. As a GitLab instance is scaled, each of these services are broken out and independently scaled according to the demands placed on them. In some cases PaaS can be leveraged for some services (e.g. Cloud Object Storage for some file systems). For the sake of redundancy some of the services become clusters of nodes storing the same data. In a horizontal configuration of GitLab there are various ancillary services required to coordinate clusters or discover of resources (e.g. PgBouncer for PostgreSQL connection management, Consul for Prometheus end point discovery). +The diagram above shows that while GitLab can be installed on a single server, it is internally composed of multiple services. As a GitLab instance is scaled, each of these services are broken out and independently scaled according to the demands placed on them. In some cases PaaS can be leveraged for some services (for example, Cloud Object Storage for some file systems). For the sake of redundancy some of the services become clusters of nodes storing the same data. In a horizontal configuration of GitLab there are various ancillary services required to coordinate clusters or discover of resources (for example, PgBouncer for PostgreSQL connection management, Consul for Prometheus end point discovery). ## Requirements @@ -106,13 +106,13 @@ performance and reliability at an increased complexity cost. ## Configure Advanced Search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). ## Cloud Native Hybrid reference architecture with Helm Charts diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index d9f349b59c7..367654b8e59 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -18,22 +18,22 @@ full list of reference architectures, see > - **Test requests per second (RPS) rates:** API: 500 RPS, Web: 50 RPS, Git (Pull): 50 RPS, Git (Push): 10 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/25k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|---------------------------------------------------|-------------|-------------------------|------------------|--------------|-----------| -| External load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | -| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|---------------------------------------------------|----------------|-------------------------|------------------|----------------|----------------| +| External load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | `D16s v3` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | +| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | | NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> @@ -1624,18 +1624,12 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. - +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. - + necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -2190,7 +2184,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -2206,10 +2200,6 @@ There are two ways of specifying object storage configuration in GitLab: Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. -GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared via NFS on any GitLab Rails and Sidekiq nodes. - -In GitLab 13.6 and later, it's also recommended to switch to [Incremental logging](../job_logs.md#incremental-logging-architecture), which uses Redis instead of disk space for temporary caching of job logs. This is required when no NFS node has been deployed. - For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides based on what features you intend to use: @@ -2239,15 +2229,21 @@ in the future. </a> </div> +## Enable incremental logging + +GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared through NFS on any GitLab Rails and Sidekiq nodes. + +While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. + ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2315,18 +2311,18 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|------------------------------------------|-------|------------------------|------------------|--------------| -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| PostgreSQL<sup>1</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | -| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|------------------------------------------|----------------|------------------------|------------------|----------------| +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| PostgreSQL<sup>1</sup> | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Internal load balancing node<sup>3</sup> | 1 | 4 vCPU, 3.6GB memory | `n1-highcpu-4` | `c5.xlarge` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Gitaly<sup>5</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | +| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index d029f356612..3b9a8f966c8 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -19,16 +19,16 @@ For a full list of reference architectures, see > - **Test requests per second (RPS) rates:** API: 40 RPS, Web: 4 RPS, Git (Pull): 4 RPS, Git (Push): 1 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/2k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|------------------------------------------|--------|-------------------------|-----------------|--------------|----------| -| Load balancer<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Redis<sup>2</sup> | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | -| Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|------------------------------------------|----------------|-------------------------|-----------------|----------------|----------------| +| Load balancer<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Redis<sup>2</sup> | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | +| Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | +| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- markdownlint-disable MD029 --> 1. Can be optionally run on reputable third-party external PaaS PostgreSQL solutions. [Google Cloud SQL](https://cloud.google.com/sql/docs/postgres/high-availability#normal) and [Amazon RDS](https://aws.amazon.com/rds/) are known to work, however Azure Database for PostgreSQL is **not recommended** due to [performance issues](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/61). Consul is primarily used for PostgreSQL high availability so can be ignored when using a PostgreSQL PaaS setup. However it is also used optionally by Prometheus for Omnibus auto host discovery. @@ -527,11 +527,9 @@ that communicate with it following the procedure described in NOTE: The self-signed certificate must specify the address you use to access the -Gitaly server. If you are addressing the Gitaly server by a hostname, you can -either use the Common Name field for this, or add it as a Subject Alternative +Gitaly server. If you are addressing the Gitaly server by a hostname, add it as a Subject Alternative Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. -[gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). It's possible to configure Gitaly servers with both an unencrypted listening address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) @@ -892,7 +890,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -943,13 +941,13 @@ in the future. ## Configure Advanced Search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -1020,12 +1018,12 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|----------------------------|-------|------------------------|-----------------|-------------| -| PostgreSQL<sup>1</sup> | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| Redis<sup>2</sup> | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | -| Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Object storage<sup>3</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|----------------------------|----------------|------------------------|-----------------|----------------| +| PostgreSQL<sup>1</sup> | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| Redis<sup>2</sup> | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | +| Gitaly | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Object storage<sup>3</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 14b8982766c..a43d33a4d4a 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -28,21 +28,21 @@ For a full list of reference architectures, see > - **Test requests per second (RPS) rates:** API: 60 RPS, Web: 6 RPS, Git (Pull): 6 RPS, Git (Push): 1 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/3k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|--------------------------------------------|-------------|-----------------------|-----------------|--------------|----------| -| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------|----------------|-----------------------|-----------------|----------------|----------------| +| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | | NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> @@ -1564,18 +1564,12 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. - +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. - + necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -2125,7 +2119,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -2141,10 +2135,6 @@ There are two ways of specifying object storage configuration in GitLab: Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. -GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared via NFS on any GitLab Rails and Sidekiq nodes. - -In GitLab 13.6 and later, it's also recommended to switch to [Incremental logging](../job_logs.md#incremental-logging-architecture), which uses Redis instead of disk space for temporary caching of job logs. This is required when no NFS node has been deployed. - For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides based on what features you intend to use: @@ -2174,15 +2164,21 @@ in the future. </a> </div> +## Enable incremental logging + +GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared through NFS on any GitLab Rails and Sidekiq nodes. + +While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. + ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2275,17 +2271,17 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|-------------------------------------------|-------|-----------------------|-----------------|-------------| -| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|-------------------------------------------|----------------|-----------------------|-----------------|----------------| +| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| PostgreSQL<sup>1</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Gitaly<sup>5</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 078e3a289cc..316969dfacc 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -18,23 +18,23 @@ full list of reference architectures, see > - **Test requests per second (RPS) rates:** API: 1000 RPS, Web: 100 RPS, Git (Pull): 100 RPS, Git (Push): 20 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/50k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|---------------------------------------------------|-------------|-------------------------|------------------|---------------|-----------| -| External load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | `D64s v3` | -| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | -| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|---------------------------------------------------|----------------|-------------------------|------------------|----------------|----------------| +| External load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | `D32s v3` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | `D64s v3` | +| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | `F32s v2` | +| Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | +| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> @@ -1633,18 +1633,12 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. - +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. - + necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -2206,7 +2200,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -2222,10 +2216,6 @@ There are two ways of specifying object storage configuration in GitLab: Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. -GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared via NFS on any GitLab Rails and Sidekiq nodes. - -In GitLab 13.6 and later, it's also recommended to switch to [Incremental logging](../job_logs.md#incremental-logging-architecture), which uses Redis instead of disk space for temporary caching of job logs. This is required when no NFS node has been deployed. - For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides based on what features you intend to use: @@ -2255,15 +2245,21 @@ in the future. </a> </div> +## Enable incremental logging + +GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared through NFS on any GitLab Rails and Sidekiq nodes. + +While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. + ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2331,18 +2327,18 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|------------------------------------------|-------|------------------------|------------------|---------------| -| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| PostgreSQL<sup>1</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | -| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | -| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|------------------------------------------|----------------|------------------------|------------------|----------------| +| Consul<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| PostgreSQL<sup>1</sup> | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Internal load balancing node<sup>3</sup> | 1 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | +| Redis/Sentinel - Cache<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Redis/Sentinel - Persistent<sup>2</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| Gitaly<sup>5</sup> | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | +| Praefect<sup>5</sup> | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index d18c77902f6..ef44d688f31 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -25,22 +25,22 @@ costly-to-operate environment by using the > - **Test requests per second (RPS) rates:** API: 100 RPS, Web: 10 RPS, Git (Pull): 10 RPS, Git (Push): 2 RPS > - **[Latest Results](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Benchmarks/Latest/5k)** -| Service | Nodes | Configuration | GCP | AWS | Azure | -|--------------------------------------------|-------------|-------------------------|-----------------|--------------|----------| -| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | -| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | `F16s v2`| -| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | n/a | -| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | +| Service | Nodes | Configuration | GCP | AWS | Azure | +|--------------------------------------------|----------------|-------------------------|-----------------|----------------|----------------| +| External load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | `D8s v3` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | +| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | `F16s v2` | +| Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | Not applicable | +| NFS server (non-Gitaly) | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | `F4s v2` | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> @@ -1562,18 +1562,12 @@ and on all Praefect clients that communicate with it following the procedure des Note the following: -- The certificate must specify the address you use to access the Praefect server. If - addressing the Praefect server by: - - - Hostname, you can either use the Common Name field for this, or add it as a Subject - Alternative Name. - - IP address, you must add it as a Subject Alternative Name to the certificate. - +- The certificate must specify the address you use to access the Praefect server. You must add the hostname or IP + address as a Subject Alternative Name to the certificate. - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. - + necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -2125,7 +2119,7 @@ GitLab has been tested on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](http://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) - [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatibility mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) @@ -2141,10 +2135,6 @@ There are two ways of specifying object storage configuration in GitLab: Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. -GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared via NFS on any GitLab Rails and Sidekiq nodes. - -In GitLab 13.6 and later, it's also recommended to switch to [Incremental logging](../job_logs.md#incremental-logging-architecture), which uses Redis instead of disk space for temporary caching of job logs. This is required when no NFS node has been deployed. - For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides based on what features you intend to use: @@ -2174,15 +2164,21 @@ in the future. </a> </div> +## Enable incremental logging + +GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily on disk in `/var/opt/gitlab/gitlab-ci/builds` by default, even when using consolidated object storage. With default configuration, this directory needs to be shared through NFS on any GitLab Rails and Sidekiq nodes. + +While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. + ## Configure Advanced Search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific data. For recommended best practices about how to set up your Elasticsearch cluster alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +[choose the optimal cluster configuration](../../integration/advanced_search/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2250,17 +2246,17 @@ future with further specific cloud provider details. Next are the backend components that run on static compute VMs via Omnibus (or External PaaS services where applicable): -| Service | Nodes | Configuration | GCP | AWS | -|-------------------------------------------|-------|-----------------------|-----------------|--------------| -| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | -| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Object storage<sup>4</sup> | n/a | n/a | n/a | n/a | +| Service | Nodes | Configuration | GCP | AWS | +|-------------------------------------------|----------------|-----------------------|-----------------|----------------| +| Redis<sup>2</sup> | 3 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| Consul<sup>1</sup> + Sentinel<sup>2</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| PostgreSQL<sup>1</sup> | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| PgBouncer<sup>1</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Internal load balancing node<sup>3</sup> | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Gitaly<sup>5</sup> | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | +| Praefect<sup>5</sup> | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Praefect PostgreSQL<sup>1</sup> | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | +| Object storage<sup>4</sup> | Not applicable | Not applicable | Not applicable | Not applicable | <!-- Disable ordered list rule https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md029---ordered-list-item-prefix --> <!-- markdownlint-disable MD029 --> diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 089e8881d7d..d9fa6c7bdd9 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -1,6 +1,6 @@ --- type: reference, concepts -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -116,7 +116,7 @@ per 1,000 users: ### How to interpret the results NOTE: -Read our blog post on [how our QA team leverages GitLab’s performance testing tool](https://about.gitlab.com/blog/2020/02/18/how-were-building-up-performance-testing-of-gitlab/). +Read our blog post on [how our QA team leverages GitLab performance testing tool](https://about.gitlab.com/blog/2020/02/18/how-were-building-up-performance-testing-of-gitlab/). Testing is done publicly and all results are shared. @@ -215,7 +215,7 @@ table.test-coverage th { ## Cost to run -The following table details the cost to run the different reference architectures across GCP, AWS, and Azure. Bare-metal costs are not included here as it varies widely depending on each customer configuration. +The following table details the cost to run the different reference architectures across GCP, AWS, and Azure. Bare-metal costs are not included here as it varies widely depending on each customer configuration. <table class="test-coverage"> <col> diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index c8c13fca59d..2d10d05da37 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/reply_by_email_postfix_setup.md b/doc/administration/reply_by_email_postfix_setup.md index aae5475312f..2652ad20329 100644 --- a/doc/administration/reply_by_email_postfix_setup.md +++ b/doc/administration/reply_by_email_postfix_setup.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/repository_storage_paths.md b/doc/administration/repository_storage_paths.md index 00aa8c214c5..3acaad8231e 100644 --- a/doc/administration/repository_storage_paths.md +++ b/doc/administration/repository_storage_paths.md @@ -174,4 +174,4 @@ information. ## Move repositories To move a repository to a different repository storage (for example, from `default` to `storage2`), use the -same process as [migrating to Gitaly Cluster](gitaly/index.md#migrating-to-gitaly-cluster). +same process as [migrating to Gitaly Cluster](gitaly/index.md#migrate-to-gitaly-cluster). diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 44bfa153123..06b6cbd4271 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -97,7 +97,7 @@ It also [restarts GitLab components](#how-to-restart-gitlab) where needed, if any of their configuration files have changed. If you manually edit any files in `/var/opt/gitlab` that are managed by Chef, -running reconfigure reverts the changes AND restarts the services that +running reconfigure reverts the changes and restarts the services that depend on those files. ## Installations from source diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 9d8bc03e5e9..a83a17d6d52 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -23,7 +23,7 @@ alternatives to server hooks include: - [Webhooks](../user/project/integrations/webhooks.md). - [GitLab CI/CD](../ci/index.md). -- [Push rules](../push_rules/push_rules.md), for a user-configurable Git hook interface. +- [Push rules](../user/project/repository/push_rules.md), for a user-configurable Git hook interface. [Geo](geo/index.md) doesn't replicate server hooks to secondary nodes. diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md index d9031e2221b..528ecf12df9 100644 --- a/doc/administration/sidekiq.md +++ b/doc/administration/sidekiq.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -233,7 +233,7 @@ To enable LDAP with the synchronization worker for Sidekiq: gitlab_rails['ldap_servers'] = { 'main' => { 'label' => 'LDAP', - 'host' => 'ldap.mydomain.com', + 'host' => 'ldap.mydomain.com', 'port' => 389, 'uid' => 'sAMAccountName', 'encryption' => 'simple_tls', @@ -269,7 +269,7 @@ To enable LDAP with the synchronization worker for Sidekiq: 'external_groups' => [], 'sync_ssh_keys' => false } - } + } gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" ``` diff --git a/doc/administration/sidekiq_health_check.md b/doc/administration/sidekiq_health_check.md index 3ff17d3682b..aaab2b7e086 100644 --- a/doc/administration/sidekiq_health_check.md +++ b/doc/administration/sidekiq_health_check.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/smime_signing_email.md b/doc/administration/smime_signing_email.md index 8fdd87a5b2d..943fd91c885 100644 --- a/doc/administration/smime_signing_email.md +++ b/doc/administration/smime_signing_email.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/system_hooks.md b/doc/administration/system_hooks.md index 71d7e7f1426..cb141a18ae6 100644 --- a/doc/administration/system_hooks.md +++ b/doc/administration/system_hooks.md @@ -61,7 +61,7 @@ To create a system hook: 1. Provide the **URL** and **Secret Token**. 1. Select the checkbox next to each optional **Trigger** you want to enable. 1. Select **Enable SSL verification**, if desired. -1. Click **Add system hook**. +1. Select **Add system hook**. ## Hooks request example diff --git a/doc/administration/timezone.md b/doc/administration/timezone.md index f4339263d34..93419751251 100644 --- a/doc/administration/timezone.md +++ b/doc/administration/timezone.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/troubleshooting/debug.md b/doc/administration/troubleshooting/debug.md index 0520ce470cb..81ca1bda5d0 100644 --- a/doc/administration/troubleshooting/debug.md +++ b/doc/administration/troubleshooting/debug.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -174,7 +174,7 @@ hearing back from the Unicorn worker. If the CPU spins to 100% while this in progress, there may be something taking longer than it should. To fix this issue, we first need to figure out what is happening. The -following tips are only recommended if you do NOT mind users being affected by +following tips are only recommended if you do not mind users being affected by downtime. Otherwise skip to the next section. 1. Load the problematic URL diff --git a/doc/administration/troubleshooting/defcon.md b/doc/administration/troubleshooting/defcon.md index a7cc47f8547..292b4b13967 100644 --- a/doc/administration/troubleshooting/defcon.md +++ b/doc/administration/troubleshooting/defcon.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/troubleshooting/diagnostics_tools.md b/doc/administration/troubleshooting/diagnostics_tools.md index 479fdb963cb..6055746238f 100644 --- a/doc/administration/troubleshooting/diagnostics_tools.md +++ b/doc/administration/troubleshooting/diagnostics_tools.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index 97e625eb0a3..7ce09252680 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -1,13 +1,16 @@ --- -stage: Enablement +stage: Data Stores group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Troubleshooting Elasticsearch **(PREMIUM SELF)** -To install and configure Elasticsearch, and for common and known issues, -visit the [administrator documentation](../../integration/elasticsearch.md). +To install and configure Elasticsearch, +visit the [administrator documentation](../../integration/advanced_search/elasticsearch.md). + +For troubleshooting, visit the +[administrator troubleshooting documentation](../../integration/advanced_search/elasticsearch_troubleshooting.md). Troubleshooting Elasticsearch requires: @@ -219,7 +222,7 @@ The output from the last command is the key here. If it shows: If all the settings look correct and it is still not using Elasticsearch for the search function, it is best to escalate to GitLab support. This could be a bug/issue. -Moving past that, it is best to attempt the same [search via the Rails console](../../integration/elasticsearch.md#i-indexed-all-the-repositories-but-i-cant-get-any-hits-for-my-search-term-in-the-ui) +Moving past that, it is best to attempt the same [search via the Rails console](../../integration/advanced_search/elasticsearch_troubleshooting.md#i-indexed-all-the-repositories-but-i-cant-get-any-hits-for-my-search-term-in-the-ui) or the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), and compare the results from what you see in GitLab. @@ -242,7 +245,7 @@ The best place to start is to determine if the issue is with creating an empty i If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch side and attempt to recreate it from the -[`recreate_index`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) +[`recreate_index`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) Rake task. If you still encounter issues, try creating an index manually on the Elasticsearch @@ -257,12 +260,12 @@ during the indexing of projects. If errors do occur, they stem from either the i - On the GitLab side. You need to rectify those. If they are not something you are familiar with, contact GitLab support for guidance. -- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/elasticsearch.md#troubleshooting). If not, speak with your Elasticsearch administrator. +- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, speak with your Elasticsearch administrator. If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks: -- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) -- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) +- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) +- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) If: @@ -375,7 +378,7 @@ If you still encounter issues after retrying the migration, reach out to GitLab ## Common issues -All common issues [should be documented](../../integration/elasticsearch.md#troubleshooting). If not, +All common issues [should be documented](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, feel free to update that page with issues you encounter and solutions. ## Replication diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 34481582d8b..b57bc0a1119 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -296,18 +296,6 @@ namespace = Namespace.find_by_full_path("<new_namespace>") ::Projects::TransferService.new(p, current_user).execute(namespace) ``` -### For Removing webhooks that is getting timeout due to large webhook logs - -```ruby -# ID is the webhook_id -hook=WebHook.find(ID) - -WebHooks::DestroyService.new(current_user).execute(hook) - -#In case the service gets timeout consider removing webhook_logs -hook.web_hook_logs.limit(BATCH_SIZE).delete_all -``` - ### Bulk update service integration password for _all_ projects For example, change the Jira user's password for all projects that have the Jira @@ -760,7 +748,7 @@ parent.members_with_descendants.count # This section lists all the groups which are pending deletion # Group.all.each do |g| - if g.marked_for_deletion? + if g.marked_for_deletion? puts "Group ID: #{g.id}" puts "Group name: #{g.name}" puts "Group path: #{g.full_path}" @@ -1458,7 +1446,7 @@ Open the rails console (`gitlab rails c`) and run the following command to see a ApplicationSetting.last.attributes ``` -Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`. +Among other attributes, the output contains all the settings available in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md), such as `elasticsearch_indexing`, `elasticsearch_url`, `elasticsearch_replicas`, and `elasticsearch_pause_indexing`. #### Setting attributes @@ -1474,7 +1462,7 @@ ApplicationSetting.last.update(elasticsearch_indexing: false) #### Getting attributes -You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/elasticsearch.md) or in the rails console by issuing: +You can then check if the settings have been set in the [Elasticsearch Integration page](../../integration/advanced_search/elasticsearch.md) or in the rails console by issuing: ```ruby Gitlab::CurrentSettings.elasticsearch_url diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md index bc9c39d57ea..7d40a9e9683 100644 --- a/doc/administration/troubleshooting/index.md +++ b/doc/administration/troubleshooting/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md index 290d6d9f21d..0c93d1ab3ee 100644 --- a/doc/administration/troubleshooting/kubernetes_cheat_sheet.md +++ b/doc/administration/troubleshooting/kubernetes_cheat_sheet.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index 913437c2d77..0245af39e45 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index 9aa490f73ef..4cc62c08f4f 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md index 91db321295d..51ef3d95a4e 100644 --- a/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md +++ b/doc/administration/troubleshooting/navigating_gitlab_via_rails_console.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index d0ed3c5c12a..cdbf786bdb2 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -67,7 +67,7 @@ This section is for links to information elsewhere in the GitLab documentation. - Required extension: `btree_gist` - Errors like this in the `production/sidekiq` log; see: - [Set default_transaction_isolation into read committed](https://docs.gitlab.com/omnibus/settings/database.html#set-default_transaction_isolation-into-read-committed): + [Set `default_transaction_isolation` into read committed](https://docs.gitlab.com/omnibus/settings/database.html#set-default_transaction_isolation-into-read-committed): ```plaintext ActiveRecord::StatementInvalid PG::TRSerializationFailure: ERROR: could not serialize access due to concurrent update @@ -138,8 +138,12 @@ idle_in_transaction_session_timeout = 60s Quoting from issue [#30528](https://gitlab.com/gitlab-org/gitlab/-/issues/30528): +<!-- vale gitlab.FutureTense = NO --> + > "If a deadlock is hit, and we resolve it through aborting the transaction after a short period, then the retry mechanisms we already have will make the deadlocked piece of work try again, and it's unlikely we'll deadlock multiple times in a row." +<!-- vale gitlab.FutureTense = YES --> + NOTE: In Support, our general approach to reconfiguring timeouts (applies also to the HTTP stack) is that it's acceptable to do it temporarily as a workaround. If it @@ -148,9 +152,9 @@ problem more completely, implement a hot fix, or make some other change that addresses the root cause. Generally, the timeouts should be put back to reasonable defaults after the root cause is resolved. -In this case, the guidance we had from development was to drop deadlock_timeout -or statement_timeout, but to leave the third setting at 60s. Setting -idle_in_transaction protects the database from sessions potentially hanging for +In this case, the guidance we had from development was to drop `deadlock_timeout` +or `statement_timeout`, but to leave the third setting at 60 seconds. Setting +`idle_in_transaction` protects the database from sessions potentially hanging for days. There's more discussion in [the issue relating to introducing this timeout on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/1053). PostgresSQL defaults: @@ -161,7 +165,7 @@ PostgresSQL defaults: Comments in issue [#30528](https://gitlab.com/gitlab-org/gitlab/-/issues/30528) indicate that these should both be set to at least a number of minutes for all Omnibus GitLab installations (so they don't hang indefinitely). However, 15s -for statement_timeout is very short, and will only be effective if the +for `statement_timeout` is very short, and is only effective if the underlying infrastructure is very performant. See current settings with: diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 7a64bcc9b87..40bfe22ac63 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/troubleshooting/ssl.md b/doc/administration/troubleshooting/ssl.md index 83df9ba19ff..d5d50127ad5 100644 --- a/doc/administration/troubleshooting/ssl.md +++ b/doc/administration/troubleshooting/ssl.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference diff --git a/doc/administration/troubleshooting/test_environments.md b/doc/administration/troubleshooting/test_environments.md index 0821f952d61..94d17ba714e 100644 --- a/doc/administration/troubleshooting/test_environments.md +++ b/doc/administration/troubleshooting/test_environments.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference @@ -26,27 +26,8 @@ but contributions are welcome. ### GitLab -Please see [our Docker test environment docs](../../install/digitaloceandocker.md#create-new-gitlab-container) -for how to run GitLab on Docker. When spinning this up with `docker-machine`, ensure -you change a few things: - -1. Update the name of the `docker-machine` host. You can see a list of hosts - with `docker-machine ls`. -1. Expose the necessary ports using the `-p` flag. Docker normally doesn't - allow access to any ports it uses outside of the container, so they must be - explicitly exposed. -1. Add any necessary `gitlab.rb` configuration to the - `GITLAB_OMNIBUS_CONFIG` variable. - -For example, when the `docker-machine` host we want to use is `do-docker`: - -```shell -docker run --detach --name gitlab \ ---env GITLAB_OMNIBUS_CONFIG="external_url 'http://$(docker-machine ip do-docker)'; gitlab_rails['gitlab_shell_ssh_port'] = 2222;" \ ---hostname $(docker-machine ip do-docker) \ --p 80:80 -p 2222:22 \ -gitlab/gitlab-ee:11.5.3-ee.0 -``` +Please see [our official Docker installation method](../../install/docker.md) +for how to run GitLab on Docker. ### SAML diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md index 3a0c6a30cde..fed3604057b 100644 --- a/doc/administration/troubleshooting/tracing_correlation_id.md +++ b/doc/administration/troubleshooting/tracing_correlation_id.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -34,7 +34,7 @@ To locate a relevant request and view its correlation ID: 1. Enable persistent logging in your network monitor. Some actions in GitLab will redirect you quickly after you submit a form, so this will help capture all relevant activity. 1. To help isolate the requests you are looking for, you can filter for `document` requests. -1. Click the request of interest to view further detail. +1. Select the request of interest to view further detail. 1. Go to the **Headers** section and look for **Response Headers**. There you should find an `x-request-id` header with a value that was randomly generated by GitLab for the request. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index d66aa5945b6..e618c4787ab 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/whats-new.md b/doc/administration/whats-new.md index d3768907989..beb47254573 100644 --- a/doc/administration/whats-new.md +++ b/doc/administration/whats-new.md @@ -21,7 +21,7 @@ feature list, the feature list is tailored to your subscription type: ## Self-managed installations -Due to our release post process, the content for **What's new** is not yet finalized +Due to our release post process, the content for **What's new** is not finalized when a new version (`.0` release) is cut. The updated **What's new** is included in the first patch release, such as `13.10.1`. diff --git a/doc/api/access_requests.md b/doc/api/access_requests.md index 08f3b78787b..32411a2f557 100644 --- a/doc/api/access_requests.md +++ b/doc/api/access_requests.md @@ -71,9 +71,9 @@ POST /groups/:id/access_requests POST /projects/:id/access_requests ``` -| Attribute | Type | Required | Description | -| --------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- |-----------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the group or project](index.md#namespaced-path-encoding) | Example request: diff --git a/doc/api/admin_sidekiq_queues.md b/doc/api/admin_sidekiq_queues.md index 079ab96c938..2b3cce80257 100644 --- a/doc/api/admin_sidekiq_queues.md +++ b/doc/api/admin_sidekiq_queues.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/broadcast_messages.md b/doc/api/broadcast_messages.md index a14de8dadd3..2252568be61 100644 --- a/doc/api/broadcast_messages.md +++ b/doc/api/broadcast_messages.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Broadcast Messages API **(FREE SELF)** -> 'target_access_levels' [introduced](https://gitlab.com/gitlab-org/growth/team-tasks/-/issues/461) in GitLab 14.8 [with a flag](../administration/feature_flags.md) named `role_targeted_broadcast_messages`. Disabled by default. +> `target_access_levels` [introduced](https://gitlab.com/gitlab-org/growth/team-tasks/-/issues/461) in GitLab 14.8 [with a flag](../administration/feature_flags.md) named `role_targeted_broadcast_messages`. Disabled by default. Broadcast messages API operates on [broadcast messages](../user/admin_area/broadcast_messages.md). @@ -100,8 +100,8 @@ Parameters: | Attribute | Type | Required | Description | |:-----------------------|:------------------|:---------|:------------------------------------------------------| | `message` | string | yes | Message to display. | -| `starts_at` | datetime | no | Starting time (defaults to current time). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | -| `ends_at` | datetime | no | Ending time (defaults to one hour from current time). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | +| `starts_at` | datetime | no | Starting time (defaults to current time in UTC). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | +| `ends_at` | datetime | no | Ending time (defaults to one hour from current time in UTC). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | | `color` | string | no | Background color hex code. | | `font` | string | no | Foreground color hex code. | | `target_access_levels` | array of integers | no | Target access levels (roles) of the broadcast message.| @@ -158,8 +158,8 @@ Parameters: |:-----------------------|:------------------|:---------|:------------------------------------------------------| | `id` | integer | yes | ID of broadcast message to update. | | `message` | string | no | Message to display. | -| `starts_at` | datetime | no | Starting time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | -| `ends_at` | datetime | no | Ending time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | +| `starts_at` | datetime | no | Starting time (UTC). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | +| `ends_at` | datetime | no | Ending time (UTC). Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | | `color` | string | no | Background color hex code. | | `font` | string | no | Foreground color hex code. | | `target_access_levels` | array of integers | no | Target access levels (roles) of the broadcast message.| @@ -205,7 +205,7 @@ Example response: Delete a broadcast message. -```shell +```plaintext DELETE /broadcast_messages/:id ``` diff --git a/doc/api/bulk_imports.md b/doc/api/bulk_imports.md index cae23b35fbe..8c5c4574206 100644 --- a/doc/api/bulk_imports.md +++ b/doc/api/bulk_imports.md @@ -27,7 +27,7 @@ POST /bulk_imports | `entities` | Array | yes | List of entities to import. | | `entities[source_type]` | String | yes | Source entity type (only `group_entity` is supported). | | `entities[source_full_path]` | String | yes | Source full path of the entity to import. | -| `entities[destination_name]` | String | yes | Destination name for the entity. | +| `entities[destination_name]` | String | yes | Destination slug for the entity. | | `entities[destination_namespace]` | String | no | Destination namespace for the entity. | ```shell @@ -41,7 +41,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla { "source_full_path": "source/full/path", "source_type": "group_entity", - "destination_name": "destination_name", + "destination_name": "destination_slug", "destination_namespace": "destination/namespace/path" } ] @@ -126,7 +126,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab "bulk_import_id": 1, "status": "finished", "source_full_path": "source_group", - "destination_name": "destination_name", + "destination_name": "destination_slug", "destination_namespace": "destination_path", "parent_id": null, "namespace_id": 1, @@ -140,7 +140,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab "bulk_import_id": 2, "status": "failed", "source_full_path": "another_group", - "destination_name": "another_name", + "destination_name": "another_slug", "destination_namespace": "another_namespace", "parent_id": null, "namespace_id": null, diff --git a/doc/api/cluster_agents.md b/doc/api/cluster_agents.md index 1c2a3b52761..16eed70fd48 100644 --- a/doc/api/cluster_agents.md +++ b/doc/api/cluster_agents.md @@ -95,7 +95,7 @@ Gets a single agent details. You must have at least the Developer role to use this endpoint. -```shell +```plaintext GET /projects/:id/cluster_agents/:agent_id ``` @@ -157,7 +157,7 @@ Registers an agent to the project. You must have at least the Maintainer role to use this endpoint. -```shell +```plaintext POST /projects/:id/cluster_agents ``` @@ -313,7 +313,7 @@ Gets a single agent token. You must have at least the Developer role to use this endpoint. -```shell +```plaintext GET /projects/:id/cluster_agents/:agent_id/tokens/:token_id ``` @@ -369,7 +369,7 @@ Creates a new token for an agent. You must have at least the Maintainer role to use this endpoint. -```shell +```plaintext POST /projects/:id/cluster_agents/:agent_id/tokens ``` @@ -380,7 +380,7 @@ Supported attributes: | `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) maintained by the authenticated user. | | `agent_id` | integer | yes | ID of the agent. | | `name` | string | yes | Name for the token. | -| `description` | string | no | Description for the token. | +| `description` | string | no | Description for the token. | Response: diff --git a/doc/api/commits.md b/doc/api/commits.md index 7be5bc3d985..3b9a72e1568 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -804,7 +804,7 @@ Example response: "discussion_locked":null, "should_remove_source_branch":null, "force_remove_source_branch":false, - "web_url":"http://https://gitlab.example.com/root/test-project/merge_requests/1", + "web_url":"https://gitlab.example.com/root/test-project/merge_requests/1", "time_stats":{ "time_estimate":0, "total_time_spent":0, diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index ee6887e9d04..adeda014af0 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -69,7 +69,7 @@ Example response: ] ``` -## List project deploy keys +## List deploy keys for project Get a list of a project's deploy keys. @@ -106,6 +106,62 @@ Example response: ] ``` +## List project deploy keys for user + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88917) in GitLab 15.1. + +Get a list of a specified user (requestee) and the authenticated user's (requester) common [project deploy keys](../user/project/deploy_keys/index.md#scope). It lists only the **enabled project keys from the common projects of requester and requestee**. + +```plaintext +GET /users/:id_or_username/project_deploy_keys +``` + +Parameters: + +| Attribute | Type | Required | Description | +|------------------- |--------|----------|------------------------------------------------------------------- | +| `id_or_username` | string | yes | The ID or username of the user to get the project deploy keys for. | + +```json +[ + { + "id": 1, + "title": "Key A", + "created_at": "2022-05-30T12:28:27.855Z", + "expires_at": null, + "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTEVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57 Key", + "fingerprint": "60:8e:10:f0:6a:82:c8:29:5f:bf:c0:38:72:00:6f:8f" + }, + { + "id": 2, + "title": "Key B", + "created_at": "2022-05-30T13:34:56.219Z", + "expires_at": null, + "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", + "fingerprint": "75:33:44:7e:55:84:dd:70:29:a3:8e:a3:c0:b9:8b:65" + } +] +``` + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/users/20/project_deploy_keys" +``` + +Example response: + +```json +[ + { + "id": 1, + "title": "Key A", + "created_at": "2022-05-30T12:28:27.855Z", + "expires_at": "2022-10-30T12:28:27.855Z", + "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTEVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57 Key", + "fingerprint": "60:8e:10:f0:6a:82:c8:29:5f:bf:c0:38:72:00:6f:8f" + } +] +``` + ## Get a single deploy key Get a single key. @@ -225,8 +281,8 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git Enables a deploy key for a project so this can be used. Returns the enabled key, with a status code 201 when successful. -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/deploy_keys/13/enable" +```plaintext +POST /projects/:id/deploy_keys/:key_id/enable ``` | Attribute | Type | Required | Description | @@ -234,6 +290,10 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `key_id` | integer | yes | The ID of the deploy key | +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/deploy_keys/12/enable" +``` + Example response: ```json diff --git a/doc/api/dora4_project_analytics.md b/doc/api/dora4_project_analytics.md index 53170c3a77f..19c7afd9d22 100644 --- a/doc/api/dora4_project_analytics.md +++ b/doc/api/dora4_project_analytics.md @@ -3,52 +3,11 @@ stage: Manage group: Optimize info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: reference, api +remove_date: '2022-05-18' +redirect_to: 'dora/metrics.md' --- -# DORA4 Analytics Project API **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/279039) in GitLab 13.7. +# DORA4 Analytics Project API (removed) **(ULTIMATE)** WARNING: -These endpoints have been removed in GitLab 14.0. Use the [DORA metrics API](dora/metrics.md) instead. - -All methods require reporter authorization. - -## List project deployment frequencies - -Get a list of all project deployment frequencies, sorted by date: - -```plaintext -GET /projects/:id/analytics/deployment_frequency?environment=:environment&from=:from&to=:to&interval=:interval -``` - -| Attribute | Type | Required | Description | -|--------------|--------|----------|-----------------------| -| `id` | string | yes | The ID of the project | -| `environment`| string | yes | The name of the environment to filter by | -| `from` | string | yes | Datetime range to start from, inclusive, ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) | -| `to` | string | no | Datetime range to end at, exclusive, ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`) | -| `interval` | string | no | The bucketing interval (`all`, `monthly`, `daily`) | - -Example request: - -```shell -curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/:id/analytics/deployment_frequency?environment=:environment&from=:from&to=:to&interval=:interval" -``` - -Example response: - -```json -[ - { - "from": "2017-01-01", - "to": "2017-01-02", - "value": 106 - }, - { - "from": "2017-01-02", - "to": "2017-01-03", - "value": 55 - } -] -``` +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/323713) in 13.11 and removed in GitLab 14.0. Use the [DORA metrics API](dora/metrics.md) instead. diff --git a/doc/api/environments.md b/doc/api/environments.md index bfc097d44ee..d67808bfd61 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -15,12 +15,12 @@ Get all environments for a given project. GET /projects/:id/environments ``` -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | --------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | no | Return the environment with this name. Mutually exclusive with `search` | -| `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name` | -| `states` | string | no | List all environments that match a specific state. Accepted values: `available` or `stopped`. If no state value given, returns all environments. | +| Attribute | Type | Required | Description | +| --------- | ------- | -------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `name` | string | no | Return the environment with this name. Mutually exclusive with `search` | +| `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name` | +| `states` | string | no | List all environments that match a specific state. Accepted values: `available`, `stopping` or `stopped`. If no state value given, returns all environments. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments?name=review%2Ffix-foo" @@ -382,10 +382,11 @@ It returns `200` if the environment was successfully stopped, and `404` if the e POST /projects/:id/environments/:environment_id/stop ``` -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | --------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | -| `environment_id` | integer | yes | The ID of the environment | +| Attribute | Type | Required | Description | +|------------------|----------------|----------|----------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `environment_id` | integer | yes | The ID of the environment | +| `force` | boolean | no | Force environment to stop without executing `on_stop` actions | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/environments/1/stop" diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index b40a3994dbf..0ad7b04d4f7 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -148,7 +148,7 @@ POST /projects/:id/feature_flags | `description` | string | no | The description of the feature flag. | | `active` | boolean | no | The active state of the flag. Defaults to true. [Supported](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38350) in GitLab 13.3 and later. | | `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | -| `strategies:name` | JSON | no | The strategy name. Can be `default`, `gradualRolloutUserId`, `userWithId`, or `gitlabUserList`. In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/36380) and later, can be [`flexibleRollout`](https://docs.getunleash.io/activation_strategy/#flexiblerollout). | +| `strategies:name` | JSON | no | The strategy name. Can be `default`, `gradualRolloutUserId`, `userWithId`, or `gitlabUserList`. In [GitLab 13.5](https://gitlab.com/gitlab-org/gitlab/-/issues/36380) and later, can be [`flexibleRollout`](https://docs.getunleash.io/user_guide/activation_strategy#gradual-rollout). | | `strategies:parameters` | JSON | no | The strategy parameters. | | `strategies:scopes` | JSON | no | The scopes for the strategy. | | `strategies:scopes:environment_scope` | string | no | The environment spec for the scope. | diff --git a/doc/api/features.md b/doc/api/features.md index 346f4879358..d4829f72958 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -127,10 +127,10 @@ POST /features/:name | `value` | integer/string | yes | `true` or `false` to enable/disable, or an integer for percentage of time | | `key` | string | no | `percentage_of_actors` or `percentage_of_time` (default) | | `feature_group` | string | no | A Feature group name | -| `user` | string | no | A GitLab username | -| `group` | string | no | A GitLab group's path, for example `gitlab-org` | -| `namespace` | string | no | A GitLab group or user namespace's path, for example `gitlab-org` or username path. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353117) in GitLab 15.0. | -| `project` | string | no | A projects path, for example `gitlab-org/gitlab-foss` | +| `user` | string | no | A GitLab username or comma-separated multiple usernames | +| `group` | string | no | A GitLab group's path, for example `gitlab-org`, or comma-separated multiple group paths | +| `namespace` | string | no | A GitLab group or user namespace's path, for example `john-doe`, or comma-separated multiple namespace paths. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353117) in GitLab 15.0. | +| `project` | string | no | A projects path, for example `gitlab-org/gitlab-foss`, or comma-separated multiple project paths | | `force` | boolean | no | Skip feature flag validation checks, such as a YAML definition | You can enable or disable a feature for a `feature_group`, a `user`, diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 93478bcf95f..fbb583f5a56 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/graphql/audit_report.md b/doc/api/graphql/audit_report.md index 76f3da2d6ea..988db29912f 100644 --- a/doc/api/graphql/audit_report.md +++ b/doc/api/graphql/audit_report.md @@ -101,7 +101,7 @@ explorer. GraphiQL explorer is available for: 1. Open the [GraphiQL explorer tool](https://gitlab.com/-/graphql-explorer). 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. -1. Click Play to get the result shown here: +1. Select **Play** to get the result shown here:  diff --git a/doc/api/graphql/custom_emoji.md b/doc/api/graphql/custom_emoji.md index 7307abc0568..e7fd9837b0f 100644 --- a/doc/api/graphql/custom_emoji.md +++ b/doc/api/graphql/custom_emoji.md @@ -81,7 +81,7 @@ explorer. GraphiQL explorer is available for: 1. Open the [GraphiQL explorer tool](https://gitlab.com/-/graphql-explorer). 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. -1. Click Play to get the result shown here: +1. Select **Play** to get the result shown here:  diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 2ad29e8cb45..09b97a78e04 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -247,11 +247,11 @@ GraphQL mutations can be detected as spam. If a mutation is detected as spam and - Use the `captchaSiteKey` to obtain a CAPTCHA response value using the appropriate CAPTCHA API. Only [Google reCAPTCHA v2](https://developers.google.com/recaptcha/docs/display) is supported. - Resubmit the request with the `X-GitLab-Captcha-Response` and `X-GitLab-Spam-Log-Id` headers set. - + NOTE: The GitLab GraphiQL implementation doesn't permit passing of headers, so we must write this as a cURL query. `--data-binary` is used to properly handle escaped double quotes -in the JSON-embedded query. +in the JSON-embedded query. ```shell export CAPTCHA_RESPONSE="<CAPTCHA response obtained from CAPTCHA service>" diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 97ef81b8bfa..805f6a506b7 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -145,6 +145,17 @@ Returns [`String!`](#string). | ---- | ---- | ----------- | | <a id="queryechotext"></a>`text` | [`String!`](#string) | Text to echo back. | +### `Query.epicBoardList` + +Returns [`EpicList`](#epiclist). + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="queryepicboardlistepicfilters"></a>`epicFilters` | [`EpicFilters`](#epicfilters) | Filters applied when getting epic metadata in the epic board list. | +| <a id="queryepicboardlistid"></a>`id` | [`BoardsEpicListID!`](#boardsepiclistid) | Global ID of the list. | + ### `Query.geoNode` Find a Geo node. @@ -545,7 +556,11 @@ Returns [`Vulnerability`](#vulnerability). ### `Query.workItem` -Find a work item. Returns `null` if `work_items` feature flag is disabled. The feature is experimental and is subject to change without notice. +Find a work item. Returns `null` if `work_items` feature flag is disabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Returns [`WorkItem`](#workitem). @@ -612,6 +627,7 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | <a id="mutationadminsidekiqqueuesdeletejobsqueuename"></a>`queueName` | [`String!`](#string) | Name of the queue to delete jobs from. | | <a id="mutationadminsidekiqqueuesdeletejobsrelatedclass"></a>`relatedClass` | [`String`](#string) | Delete jobs matching related_class in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsremoteip"></a>`remoteIp` | [`String`](#string) | Delete jobs matching remote_ip in the context metadata. | +| <a id="mutationadminsidekiqqueuesdeletejobsrootcallerid"></a>`rootCallerId` | [`String`](#string) | Delete jobs matching root_caller_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsrootnamespace"></a>`rootNamespace` | [`String`](#string) | Delete jobs matching root_namespace in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobssubscriptionplan"></a>`subscriptionPlan` | [`String`](#string) | Delete jobs matching subscription_plan in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsuser"></a>`user` | [`String`](#string) | Delete jobs matching user in the context metadata. | @@ -673,6 +689,10 @@ Input type: `AlertTodoCreateInput` ### `Mutation.apiFuzzingCiConfigurationCreate` +WARNING: +**Deprecated** in 15.1. +The configuration snippet is now generated client-side. + Input type: `ApiFuzzingCiConfigurationCreateInput` #### Arguments @@ -697,6 +717,45 @@ Input type: `ApiFuzzingCiConfigurationCreateInput` | <a id="mutationapifuzzingciconfigurationcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationapifuzzingciconfigurationcreategitlabciyamleditpath"></a>`gitlabCiYamlEditPath` **{warning-solid}** | [`String`](#string) | **Deprecated:** The configuration snippet is now generated client-side. Deprecated in 14.6. | +### `Mutation.auditEventsStreamingHeadersCreate` + +Input type: `AuditEventsStreamingHeadersCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingheaderscreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingheaderscreatedestinationid"></a>`destinationId` | [`AuditEventsExternalAuditEventDestinationID!`](#auditeventsexternalauditeventdestinationid) | Destination to associate header with. | +| <a id="mutationauditeventsstreamingheaderscreatekey"></a>`key` | [`String!`](#string) | Header key. | +| <a id="mutationauditeventsstreamingheaderscreatevalue"></a>`value` | [`String!`](#string) | Header value. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingheaderscreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingheaderscreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationauditeventsstreamingheaderscreateheader"></a>`header` | [`AuditEventStreamingHeader`](#auditeventstreamingheader) | Created header. | + +### `Mutation.auditEventsStreamingHeadersDestroy` + +Input type: `AuditEventsStreamingHeadersDestroyInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingheadersdestroyclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingheadersdestroyheaderid"></a>`headerId` | [`AuditEventsStreamingHeaderID!`](#auditeventsstreamingheaderid) | Header to delete. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationauditeventsstreamingheadersdestroyclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationauditeventsstreamingheadersdestroyerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.awardEmojiAdd` Input type: `AwardEmojiAddInput` @@ -2231,6 +2290,25 @@ Input type: `DestroyPackageFileInput` | <a id="mutationdestroypackagefileclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationdestroypackagefileerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.destroyPackageFiles` + +Input type: `DestroyPackageFilesInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationdestroypackagefilesclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationdestroypackagefilesids"></a>`ids` | [`[PackagesPackageFileID!]!`](#packagespackagefileid) | IDs of the Package file. | +| <a id="mutationdestroypackagefilesprojectpath"></a>`projectPath` | [`ID!`](#id) | Project path where the packages cleanup policy is located. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationdestroypackagefilesclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationdestroypackagefileserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.destroySnippet` Input type: `DestroySnippetInput` @@ -2804,6 +2882,28 @@ Input type: `HttpIntegrationUpdateInput` | <a id="mutationhttpintegrationupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationhttpintegrationupdateintegration"></a>`integration` | [`AlertManagementHttpIntegration`](#alertmanagementhttpintegration) | HTTP integration. | +### `Mutation.issuableResourceLinkCreate` + +Input type: `IssuableResourceLinkCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissuableresourcelinkcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissuableresourcelinkcreateid"></a>`id` | [`IssueID!`](#issueid) | Incident id to associate the resource link with. | +| <a id="mutationissuableresourcelinkcreatelink"></a>`link` | [`String!`](#string) | Link of the resource. | +| <a id="mutationissuableresourcelinkcreatelinktext"></a>`linkText` | [`String`](#string) | Link text of the resource. | +| <a id="mutationissuableresourcelinkcreatelinktype"></a>`linkType` | [`IssuableResourceLinkType`](#issuableresourcelinktype) | Link type of the resource. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationissuableresourcelinkcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationissuableresourcelinkcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationissuableresourcelinkcreateissuableresourcelink"></a>`issuableResourceLink` | [`IssuableResourceLink`](#issuableresourcelink) | Issuable resource link. | + ### `Mutation.issueMove` Input type: `IssueMoveInput` @@ -4144,6 +4244,7 @@ Input type: `ReleaseCreateInput` | <a id="mutationreleasecreateprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project the release is associated with. | | <a id="mutationreleasecreateref"></a>`ref` | [`String`](#string) | Commit SHA or branch name to use if creating a new tag. | | <a id="mutationreleasecreatereleasedat"></a>`releasedAt` | [`Time`](#time) | Date and time for the release. Defaults to the current date and time. | +| <a id="mutationreleasecreatetagmessage"></a>`tagMessage` | [`String`](#string) | Message to use if creating a new annotated tag. | | <a id="mutationreleasecreatetagname"></a>`tagName` | [`String!`](#string) | Name of the tag to associate with the release. | #### Fields @@ -4270,6 +4371,7 @@ Input type: `RunnerUpdateInput` | <a id="mutationrunnerupdatedescription"></a>`description` | [`String`](#string) | Description of the runner. | | <a id="mutationrunnerupdateid"></a>`id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner to update. | | <a id="mutationrunnerupdatelocked"></a>`locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | +| <a id="mutationrunnerupdatemaintenancenote"></a>`maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | | <a id="mutationrunnerupdatemaximumtimeout"></a>`maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | | <a id="mutationrunnerupdatepaused"></a>`paused` | [`Boolean`](#boolean) | Indicates the runner is not allowed to receive jobs. | | <a id="mutationrunnerupdateprivateprojectsminutescostfactor"></a>`privateProjectsMinutesCostFactor` | [`Float`](#float) | Private projects' "minutes cost factor" associated with the runner (GitLab.com only). | @@ -5090,6 +5192,26 @@ Input type: `UpdateNoteInput` | <a id="mutationupdatenoteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationupdatenotenote"></a>`note` | [`Note`](#note) | Note after mutation. | +### `Mutation.updatePackagesCleanupPolicy` + +Input type: `UpdatePackagesCleanupPolicyInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationupdatepackagescleanuppolicyclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationupdatepackagescleanuppolicykeepnduplicatedpackagefiles"></a>`keepNDuplicatedPackageFiles` | [`PackagesCleanupKeepDuplicatedPackageFilesEnum`](#packagescleanupkeepduplicatedpackagefilesenum) | Number of duplicated package files to retain. | +| <a id="mutationupdatepackagescleanuppolicyprojectpath"></a>`projectPath` | [`ID!`](#id) | Project path where the packages cleanup policy is located. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationupdatepackagescleanuppolicyclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationupdatepackagescleanuppolicyerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationupdatepackagescleanuppolicypackagescleanuppolicy"></a>`packagesCleanupPolicy` | [`PackagesCleanupPolicy`](#packagescleanuppolicy) | Packages cleanup policy after mutation. | + ### `Mutation.updateRequirement` Input type: `UpdateRequirementInput` @@ -5347,7 +5469,11 @@ Input type: `VulnerabilityRevertToDetectedInput` ### `Mutation.workItemCreate` -Creates a work item. Available only when feature flag `work_items` is enabled. The feature is experimental and is subject to change without notice. +Creates a work item. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Input type: `WorkItemCreateInput` @@ -5371,7 +5497,11 @@ Input type: `WorkItemCreateInput` ### `Mutation.workItemCreateFromTask` -Creates a work item from a task in another work item's description. Available only when feature flag `work_items` is enabled. This feature is experimental and is subject to change without notice. +Creates a work item from a task in another work item's description. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Input type: `WorkItemCreateFromTaskInput` @@ -5394,7 +5524,11 @@ Input type: `WorkItemCreateFromTaskInput` ### `Mutation.workItemDelete` -Deletes a work item. Available only when feature flag `work_items` is enabled. The feature is experimental and is subject to change without notice. +Deletes a work item. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Input type: `WorkItemDeleteInput` @@ -5415,7 +5549,11 @@ Input type: `WorkItemDeleteInput` ### `Mutation.workItemDeleteTask` -Deletes a task in a work item's description. Available only when feature flag `work_items` is enabled. This feature is experimental and is subject to change without notice. +Deletes a task in a work item's description. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Input type: `WorkItemDeleteTaskInput` @@ -5438,7 +5576,11 @@ Input type: `WorkItemDeleteTaskInput` ### `Mutation.workItemUpdate` -Updates a work item by Global ID. Available only when feature flag `work_items` is enabled. The feature is experimental and is subject to change without notice. +Updates a work item by Global ID. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. Input type: `WorkItemUpdateInput` @@ -5459,6 +5601,59 @@ Input type: `WorkItemUpdateInput` | <a id="mutationworkitemupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationworkitemupdateworkitem"></a>`workItem` | [`WorkItem`](#workitem) | Updated work item. | +### `Mutation.workItemUpdateTask` + +Updates a work item's task by Global ID. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. + +Input type: `WorkItemUpdateTaskInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemupdatetaskclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemupdatetaskid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| <a id="mutationworkitemupdatetasktaskdata"></a>`taskData` | [`WorkItemUpdatedTaskInput!`](#workitemupdatedtaskinput) | Arguments necessary to update a task. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemupdatetaskclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemupdatetaskerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationworkitemupdatetasktask"></a>`task` | [`WorkItem`](#workitem) | Updated task. | +| <a id="mutationworkitemupdatetaskworkitem"></a>`workItem` | [`WorkItem`](#workitem) | Updated work item. | + +### `Mutation.workItemUpdateWidgets` + +Updates the attributes of a work item's widgets by global ID. Available only when feature flag `work_items` is enabled. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. + +Input type: `WorkItemUpdateWidgetsInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemupdatewidgetsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemupdatewidgetsdescriptionwidget"></a>`descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | +| <a id="mutationworkitemupdatewidgetsid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationworkitemupdatewidgetsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemupdatewidgetserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationworkitemupdatewidgetsworkitem"></a>`workItem` | [`WorkItem`](#workitem) | Updated work item. | + ## Connections Some types in our schema are `Connection` types - they represent a paginated @@ -5590,6 +5785,29 @@ The edge type for [`AlertManagementIntegration`](#alertmanagementintegration). | <a id="alertmanagementintegrationedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="alertmanagementintegrationedgenode"></a>`node` | [`AlertManagementIntegration`](#alertmanagementintegration) | The item at the end of the edge. | +#### `AuditEventStreamingHeaderConnection` + +The connection type for [`AuditEventStreamingHeader`](#auditeventstreamingheader). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="auditeventstreamingheaderconnectionedges"></a>`edges` | [`[AuditEventStreamingHeaderEdge]`](#auditeventstreamingheaderedge) | A list of edges. | +| <a id="auditeventstreamingheaderconnectionnodes"></a>`nodes` | [`[AuditEventStreamingHeader]`](#auditeventstreamingheader) | A list of nodes. | +| <a id="auditeventstreamingheaderconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `AuditEventStreamingHeaderEdge` + +The edge type for [`AuditEventStreamingHeader`](#auditeventstreamingheader). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="auditeventstreamingheaderedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="auditeventstreamingheaderedgenode"></a>`node` | [`AuditEventStreamingHeader`](#auditeventstreamingheader) | The item at the end of the edge. | + #### `AwardEmojiConnection` The connection type for [`AwardEmoji`](#awardemoji). @@ -5874,11 +6092,24 @@ The connection type for [`CiJob`](#cijob). | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="cijobconnectioncount"></a>`count` | [`Int!`](#int) | Total count of collection. | | <a id="cijobconnectionedges"></a>`edges` | [`[CiJobEdge]`](#cijobedge) | A list of edges. | | <a id="cijobconnectionnodes"></a>`nodes` | [`[CiJob]`](#cijob) | A list of nodes. | | <a id="cijobconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +##### Fields with arguments + +###### `CiJobConnection.count` + +Limited count of collection. Returns limit + 1 for counts greater than the limit. + +Returns [`Int!`](#int). + +####### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cijobconnectioncountlimit"></a>`limit` | [`Int`](#int) | Limit value to be applied to the count query. Default is 1000. | + #### `CiJobEdge` The edge type for [`CiJob`](#cijob). @@ -7710,6 +7941,7 @@ The connection type for [`Project`](#project). | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="projectconnectioncount"></a>`count` | [`Int!`](#int) | Total count of collection. | | <a id="projectconnectionedges"></a>`edges` | [`[ProjectEdge]`](#projectedge) | A list of edges. | | <a id="projectconnectionnodes"></a>`nodes` | [`[Project]`](#project) | A list of nodes. | | <a id="projectconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | @@ -8662,6 +8894,29 @@ The connection type for [`Vulnerability`](#vulnerability). | <a id="vulnerabilityconnectionnodes"></a>`nodes` | [`[Vulnerability]`](#vulnerability) | A list of nodes. | | <a id="vulnerabilityconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +#### `VulnerabilityContainerImageConnection` + +The connection type for [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="vulnerabilitycontainerimageconnectionedges"></a>`edges` | [`[VulnerabilityContainerImageEdge]`](#vulnerabilitycontainerimageedge) | A list of edges. | +| <a id="vulnerabilitycontainerimageconnectionnodes"></a>`nodes` | [`[VulnerabilityContainerImage]`](#vulnerabilitycontainerimage) | A list of nodes. | +| <a id="vulnerabilitycontainerimageconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `VulnerabilityContainerImageEdge` + +The edge type for [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="vulnerabilitycontainerimageedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="vulnerabilitycontainerimageedgenode"></a>`node` | [`VulnerabilityContainerImage`](#vulnerabilitycontainerimage) | The item at the end of the edge. | + #### `VulnerabilityEdge` The edge type for [`Vulnerability`](#vulnerability). @@ -8742,6 +8997,29 @@ The edge type for [`VulnerabilityScanner`](#vulnerabilityscanner). | <a id="vulnerabilityscanneredgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="vulnerabilityscanneredgenode"></a>`node` | [`VulnerabilityScanner`](#vulnerabilityscanner) | The item at the end of the edge. | +#### `WorkItemConnection` + +The connection type for [`WorkItem`](#workitem). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemconnectionedges"></a>`edges` | [`[WorkItemEdge]`](#workitemedge) | A list of edges. | +| <a id="workitemconnectionnodes"></a>`nodes` | [`[WorkItem]`](#workitem) | A list of nodes. | +| <a id="workitemconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `WorkItemEdge` + +The edge type for [`WorkItem`](#workitem). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="workitemedgenode"></a>`node` | [`WorkItem`](#workitem) | The item at the end of the edge. | + #### `WorkItemTypeConnection` The connection type for [`WorkItemType`](#workitemtype). @@ -9001,6 +9279,18 @@ Represents a vulnerability asset type. | <a id="assettypetype"></a>`type` | [`String!`](#string) | Type of the asset. | | <a id="assettypeurl"></a>`url` | [`String!`](#string) | URL of the asset. | +### `AuditEventStreamingHeader` + +Represents a HTTP header key/value that belongs to an audit streaming destination. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="auditeventstreamingheaderid"></a>`id` | [`ID!`](#id) | ID of the header. | +| <a id="auditeventstreamingheaderkey"></a>`key` | [`String!`](#string) | Key of the header. | +| <a id="auditeventstreamingheadervalue"></a>`value` | [`String!`](#string) | Value of the header. | + ### `AwardEmoji` An emoji awarded by a user. @@ -9127,6 +9417,10 @@ Represents an epic on an issue board. | ---- | ---- | ----------- | | <a id="boardepicauthor"></a>`author` | [`UserCore!`](#usercore) | Author of the epic. | | <a id="boardepicawardemoji"></a>`awardEmoji` | [`AwardEmojiConnection`](#awardemojiconnection) | List of award emojis associated with the epic. (see [Connections](#connections)) | +| <a id="boardepicblocked"></a>`blocked` | [`Boolean`](#boolean) | Indicates the epic is blocked. | +| <a id="boardepicblockedbycount"></a>`blockedByCount` | [`Int`](#int) | Count of epics blocking this epic. | +| <a id="boardepicblockedbyepics"></a>`blockedByEpics` | [`EpicConnection`](#epicconnection) | Epics blocking this epic. (see [Connections](#connections)) | +| <a id="boardepicblockingcount"></a>`blockingCount` | [`Int`](#int) | Count of epics that this epic is blocking. | | <a id="boardepicclosedat"></a>`closedAt` | [`Time`](#time) | Timestamp of when the epic was closed. | | <a id="boardepiccolor"></a>`color` | [`String!`](#string) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | | <a id="boardepicconfidential"></a>`confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | @@ -9212,6 +9506,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="boardepicancestorsstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="boardepicancestorsstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="boardepicancestorstimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="boardepicancestorstoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="boardepicancestorsupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="boardepicancestorsupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -9249,6 +9544,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="boardepicchildrenstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="boardepicchildrenstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="boardepicchildrentimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="boardepicchildrentoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="boardepicchildrenupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="boardepicchildrenupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -9568,7 +9864,9 @@ Represents the total number of issues and their weights for a particular day. | <a id="cirunnerjobcount"></a>`jobCount` | [`Int`](#int) | Number of jobs processed by the runner (limited to 1000, plus one to indicate that more items exist). | | <a id="cirunnerlocked"></a>`locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | | <a id="cirunnermaintenancenote"></a>`maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | +| <a id="cirunnermaintenancenotehtml"></a>`maintenanceNoteHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `maintenance_note`. | | <a id="cirunnermaximumtimeout"></a>`maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | +| <a id="cirunnerownerproject"></a>`ownerProject` | [`Project`](#project) | Project that owns the runner. For project runners only. | | <a id="cirunnerpaused"></a>`paused` | [`Boolean!`](#boolean) | Indicates the runner is paused and not available to run jobs. | | <a id="cirunnerplatformname"></a>`platformName` | [`String`](#string) | Platform provided by the runner. | | <a id="cirunnerprivateprojectsminutescostfactor"></a>`privateProjectsMinutesCostFactor` | [`Float`](#float) | Private projects' "minutes cost factor" associated with the runner (GitLab.com only). | @@ -9581,7 +9879,7 @@ Represents the total number of issues and their weights for a particular day. | <a id="cirunnershortsha"></a>`shortSha` | [`String`](#string) | First eight characters of the runner's token used to authenticate new job requests. Used as the runner's unique ID. | | <a id="cirunnertaglist"></a>`tagList` | [`[String!]`](#string) | Tags associated with the runner. | | <a id="cirunnertokenexpiresat"></a>`tokenExpiresAt` | [`Time`](#time) | Runner token expiration time. | -| <a id="cirunnerupgradestatus"></a>`upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatusType`](#cirunnerupgradestatustype) | **Deprecated** in 14.10. This feature is in Alpha, and can be removed or changed at any point. | +| <a id="cirunnerupgradestatus"></a>`upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatusType`](#cirunnerupgradestatustype) | **Introduced** in 14.10. This feature is in Alpha. It can be changed or removed at any time. | | <a id="cirunneruserpermissions"></a>`userPermissions` | [`RunnerPermissions!`](#runnerpermissions) | Permissions for the current user on the resource. | | <a id="cirunnerversion"></a>`version` | [`String`](#string) | Version of the runner. | @@ -10704,6 +11002,10 @@ Represents an epic. | ---- | ---- | ----------- | | <a id="epicauthor"></a>`author` | [`UserCore!`](#usercore) | Author of the epic. | | <a id="epicawardemoji"></a>`awardEmoji` | [`AwardEmojiConnection`](#awardemojiconnection) | List of award emojis associated with the epic. (see [Connections](#connections)) | +| <a id="epicblocked"></a>`blocked` | [`Boolean`](#boolean) | Indicates the epic is blocked. | +| <a id="epicblockedbycount"></a>`blockedByCount` | [`Int`](#int) | Count of epics blocking this epic. | +| <a id="epicblockedbyepics"></a>`blockedByEpics` | [`EpicConnection`](#epicconnection) | Epics blocking this epic. (see [Connections](#connections)) | +| <a id="epicblockingcount"></a>`blockingCount` | [`Int`](#int) | Count of epics that this epic is blocking. | | <a id="epicclosedat"></a>`closedAt` | [`Time`](#time) | Timestamp of when the epic was closed. | | <a id="epiccolor"></a>`color` | [`String!`](#string) | Color of the epic. Available only when feature flag `epic_color_highlight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | | <a id="epicconfidential"></a>`confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | @@ -10788,6 +11090,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="epicancestorsstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="epicancestorsstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="epicancestorstimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="epicancestorstoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="epicancestorsupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="epicancestorsupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -10825,6 +11128,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="epicchildrenstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="epicchildrenstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="epicchildrentimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="epicchildrentoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="epicchildrenupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="epicchildrenupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -10942,6 +11246,7 @@ Relationship between an epic and an issue. | <a id="epicissueblockedbycount"></a>`blockedByCount` | [`Int`](#int) | Count of issues blocking this issue. | | <a id="epicissueblockedbyissues"></a>`blockedByIssues` | [`IssueConnection`](#issueconnection) | Issues blocking this issue. (see [Connections](#connections)) | | <a id="epicissueblockingcount"></a>`blockingCount` | [`Int!`](#int) | Count of issues this issue is blocking. | +| <a id="epicissueclosedasduplicateof"></a>`closedAsDuplicateOf` | [`Issue`](#issue) | Issue this issue was closed as a duplicate of. | | <a id="epicissueclosedat"></a>`closedAt` | [`Time`](#time) | Timestamp of when the issue was closed. | | <a id="epicissueconfidential"></a>`confidential` | [`Boolean!`](#boolean) | Indicates the issue is confidential. | | <a id="epicissuecreatenoteemail"></a>`createNoteEmail` | [`String`](#string) | User specific email address for the issue. | @@ -11143,6 +11448,7 @@ Represents an external resource to send audit events to. | ---- | ---- | ----------- | | <a id="externalauditeventdestinationdestinationurl"></a>`destinationUrl` | [`String!`](#string) | External destination to send audit events to. | | <a id="externalauditeventdestinationgroup"></a>`group` | [`Group!`](#group) | Group the destination belongs to. | +| <a id="externalauditeventdestinationheaders"></a>`headers` | [`AuditEventStreamingHeaderConnection!`](#auditeventstreamingheaderconnection) | List of additional HTTP headers sent with each event. Available only when feature flag `streaming_audit_event_headers` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. (see [Connections](#connections)) | | <a id="externalauditeventdestinationid"></a>`id` | [`ID!`](#id) | ID of the destination. | | <a id="externalauditeventdestinationverificationtoken"></a>`verificationToken` | [`String!`](#string) | Verification token to validate source of event. | @@ -11369,7 +11675,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean!`](#boolean) | Indicates whether to regularly prune stale group runners. Defaults to false. | | <a id="groupautodevopsenabled"></a>`autoDevopsEnabled` | [`Boolean`](#boolean) | Indicates whether Auto DevOps is enabled for all projects within this group. | | <a id="groupavatarurl"></a>`avatarUrl` | [`String`](#string) | Avatar URL of the group. | -| <a id="groupcontacts"></a>`contacts` | [`CustomerRelationsContactConnection`](#customerrelationscontactconnection) | Find contacts of this group. (see [Connections](#connections)) | | <a id="groupcontainerrepositoriescount"></a>`containerRepositoriesCount` | [`Int!`](#int) | Number of container repositories in the group. | | <a id="groupcontainslockedprojects"></a>`containsLockedProjects` | [`Boolean!`](#boolean) | Includes at least one project where the repository size exceeds the limit. | | <a id="groupcrossprojectpipelineavailable"></a>`crossProjectPipelineAvailable` | [`Boolean!`](#boolean) | Indicates if the cross_project_pipeline feature is available for the namespace. | @@ -11396,7 +11701,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="grouplfsenabled"></a>`lfsEnabled` | [`Boolean`](#boolean) | Indicates if Large File Storage (LFS) is enabled for namespace. | | <a id="groupmentionsdisabled"></a>`mentionsDisabled` | [`Boolean`](#boolean) | Indicates if a group is disabled from getting mentioned. | | <a id="groupname"></a>`name` | [`String!`](#string) | Name of the namespace. | -| <a id="grouporganizations"></a>`organizations` | [`CustomerRelationsOrganizationConnection`](#customerrelationsorganizationconnection) | Find organizations of this group. (see [Connections](#connections)) | | <a id="grouppackagesettings"></a>`packageSettings` | [`PackageSettings`](#packagesettings) | Package settings for the namespace. | | <a id="groupparent"></a>`parent` | [`Group`](#group) | Parent group. | | <a id="grouppath"></a>`path` | [`String!`](#string) | Path of the namespace. | @@ -11494,6 +11798,24 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="groupcomplianceframeworksid"></a>`id` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | Global ID of a specific compliance framework to return. | +##### `Group.contacts` + +Find contacts of this group. + +Returns [`CustomerRelationsContactConnection`](#customerrelationscontactconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="groupcontactsids"></a>`ids` | [`[CustomerRelationsContactID!]`](#customerrelationscontactid) | Filter contacts by IDs. | +| <a id="groupcontactssearch"></a>`search` | [`String`](#string) | Search term to find contacts with. | +| <a id="groupcontactsstate"></a>`state` | [`CustomerRelationsContactState`](#customerrelationscontactstate) | State of the contacts to search for. | + ##### `Group.containerRepositories` Container repositories of the group. @@ -11559,6 +11881,7 @@ Returns [`Epic`](#epic). | <a id="groupepicstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="groupepicstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="groupepictimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="groupepictoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="groupepicupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="groupepicupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -11608,6 +11931,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupepicsstartdate"></a>`startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | <a id="groupepicsstate"></a>`state` | [`EpicState`](#epicstate) | Filter epics by state. | | <a id="groupepicstimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | +| <a id="groupepicstoplevelhierarchyonly"></a>`topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | | <a id="groupepicsupdatedafter"></a>`updatedAfter` | [`Time`](#time) | Epics updated after this date. | | <a id="groupepicsupdatedbefore"></a>`updatedBefore` | [`Time`](#time) | Epics updated before this date. | @@ -11831,6 +12155,24 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupmilestonestimeframe"></a>`timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | <a id="groupmilestonestitle"></a>`title` | [`String`](#string) | Title of the milestone. | +##### `Group.organizations` + +Find organizations of this group. + +Returns [`CustomerRelationsOrganizationConnection`](#customerrelationsorganizationconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="grouporganizationsids"></a>`ids` | [`[CustomerRelationsOrganizationID!]`](#customerrelationsorganizationid) | Filter organizations by IDs. | +| <a id="grouporganizationssearch"></a>`search` | [`String`](#string) | Search term used to find organizations with. | +| <a id="grouporganizationsstate"></a>`state` | [`CustomerRelationsOrganizationState`](#customerrelationsorganizationstate) | State of the organization to search for. | + ##### `Group.packages` Packages of the group. @@ -12039,7 +12381,7 @@ Represents a Group Membership. | <a id="groupmemberexpiresat"></a>`expiresAt` | [`Time`](#time) | Date and time the membership expires. | | <a id="groupmembergroup"></a>`group` | [`Group`](#group) | Group that a User is a member of. | | <a id="groupmemberid"></a>`id` | [`ID!`](#id) | ID of the member. | -| <a id="groupmembernotificationemail"></a>`notificationEmail` | [`String`](#string) | Group notification email for User. Only availble for admins. | +| <a id="groupmembernotificationemail"></a>`notificationEmail` | [`String`](#string) | Group notification email for User. Only available for admins. | | <a id="groupmemberupdatedat"></a>`updatedAt` | [`Time`](#time) | Date and time the membership was last updated. | | <a id="groupmemberuser"></a>`user` | [`UserCore`](#usercore) | User that is associated with the member object. | | <a id="groupmemberuserpermissions"></a>`userPermissions` | [`GroupPermissions!`](#grouppermissions) | Permissions for the current user on the resource. | @@ -12252,6 +12594,20 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | <a id="instancesecuritydashboardvulnerabilityseveritiescountseverity"></a>`severity` | [`[VulnerabilitySeverity!]`](#vulnerabilityseverity) | Filter vulnerabilities by severity. | | <a id="instancesecuritydashboardvulnerabilityseveritiescountstate"></a>`state` | [`[VulnerabilityState!]`](#vulnerabilitystate) | Filter vulnerabilities by state. | +### `IssuableResourceLink` + +Describes an issuable resource link for incident issues. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="issuableresourcelinkid"></a>`id` | [`IncidentManagementIssuableResourceLinkID!`](#incidentmanagementissuableresourcelinkid) | ID of the Issuable resource link. | +| <a id="issuableresourcelinkissue"></a>`issue` | [`Issue!`](#issue) | Incident of the resource link. | +| <a id="issuableresourcelinklink"></a>`link` | [`String!`](#string) | Web Link to the resource. | +| <a id="issuableresourcelinklinktext"></a>`linkText` | [`String`](#string) | Optional text for the link. | +| <a id="issuableresourcelinklinktype"></a>`linkType` | [`IssuableResourceLinkType!`](#issuableresourcelinktype) | Type of the resource link. | + ### `Issue` #### Fields @@ -12265,6 +12621,7 @@ Returns [`VulnerabilitySeveritiesCount`](#vulnerabilityseveritiescount). | <a id="issueblockedbycount"></a>`blockedByCount` | [`Int`](#int) | Count of issues blocking this issue. | | <a id="issueblockedbyissues"></a>`blockedByIssues` | [`IssueConnection`](#issueconnection) | Issues blocking this issue. (see [Connections](#connections)) | | <a id="issueblockingcount"></a>`blockingCount` | [`Int!`](#int) | Count of issues this issue is blocking. | +| <a id="issueclosedasduplicateof"></a>`closedAsDuplicateOf` | [`Issue`](#issue) | Issue this issue was closed as a duplicate of. | | <a id="issueclosedat"></a>`closedAt` | [`Time`](#time) | Timestamp of when the issue was closed. | | <a id="issueconfidential"></a>`confidential` | [`Boolean!`](#boolean) | Indicates the issue is confidential. | | <a id="issuecreatenoteemail"></a>`createNoteEmail` | [`String`](#string) | User specific email address for the issue. | @@ -13812,6 +14169,7 @@ Represents a milestone. | <a id="milestoneid"></a>`id` | [`ID!`](#id) | ID of the milestone. | | <a id="milestoneiid"></a>`iid` | [`ID!`](#id) | Internal ID of the milestone. | | <a id="milestoneprojectmilestone"></a>`projectMilestone` | [`Boolean!`](#boolean) | Indicates if milestone is at project level. | +| <a id="milestonereleases"></a>`releases` | [`ReleaseConnection`](#releaseconnection) | Releases associated with this milestone. (see [Connections](#connections)) | | <a id="milestonestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the milestone start date. | | <a id="milestonestate"></a>`state` | [`MilestoneStateEnum!`](#milestonestateenum) | State of the milestone. | | <a id="milestonestats"></a>`stats` | [`MilestoneStats`](#milestonestats) | Milestone statistics. | @@ -14271,6 +14629,17 @@ Represents a package tag. | <a id="packagetagname"></a>`name` | [`String!`](#string) | Name of the tag. | | <a id="packagetagupdatedat"></a>`updatedAt` | [`Time!`](#time) | Updated date. | +### `PackagesCleanupPolicy` + +A packages cleanup policy designed to keep only packages and packages assets that matter most. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="packagescleanuppolicykeepnduplicatedpackagefiles"></a>`keepNDuplicatedPackageFiles` | [`PackagesCleanupKeepDuplicatedPackageFilesEnum!`](#packagescleanupkeepduplicatedpackagefilesenum) | Number of duplicated package files to retain. | +| <a id="packagescleanuppolicynextrunat"></a>`nextRunAt` | [`Time`](#time) | Next time that this packages cleanup policy will be executed. | + ### `PageInfo` Information about pagination in a connection. @@ -14338,6 +14707,7 @@ Represents a file or directory in the project repository that has been locked. | <a id="pipelineid"></a>`id` | [`ID!`](#id) | ID of the pipeline. | | <a id="pipelineiid"></a>`iid` | [`String!`](#string) | Internal ID of the pipeline. | | <a id="pipelinejobartifacts"></a>`jobArtifacts` | [`[CiJobArtifact!]`](#cijobartifact) | Job artifacts of the pipeline. | +| <a id="pipelinemergerequesteventtype"></a>`mergeRequestEventType` | [`PipelineMergeRequestEventType`](#pipelinemergerequesteventtype) | Event type of the pipeline associated with a merge request. | | <a id="pipelinepath"></a>`path` | [`String`](#string) | Relative path to the pipeline's page. | | <a id="pipelineproject"></a>`project` | [`Project`](#project) | Project the pipeline belongs to. | | <a id="pipelinequeuedduration"></a>`queuedDuration` | [`Duration`](#duration) | How long the pipeline was queued before starting. | @@ -14516,9 +14886,9 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="pipelinesecurityreportfindingidentifiers"></a>`identifiers` | [`[VulnerabilityIdentifier!]!`](#vulnerabilityidentifier) | Identifiers of the vulnerability finding. | | <a id="pipelinesecurityreportfindinglinks"></a>`links` | [`[VulnerabilityLink!]`](#vulnerabilitylink) | List of links associated with the vulnerability. | | <a id="pipelinesecurityreportfindinglocation"></a>`location` | [`VulnerabilityLocation`](#vulnerabilitylocation) | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability. | -| <a id="pipelinesecurityreportfindingname"></a>`name` | [`String`](#string) | Name of the vulnerability finding. | +| <a id="pipelinesecurityreportfindingname"></a>`name` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. Use `title`. | | <a id="pipelinesecurityreportfindingproject"></a>`project` | [`Project`](#project) | Project on which the vulnerability finding was found. | -| <a id="pipelinesecurityreportfindingprojectfingerprint"></a>`projectFingerprint` | [`String`](#string) | Name of the vulnerability finding. | +| <a id="pipelinesecurityreportfindingprojectfingerprint"></a>`projectFingerprint` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. The `project_fingerprint` attribute is being deprecated. Use `uuid` to identify findings. | | <a id="pipelinesecurityreportfindingreporttype"></a>`reportType` | [`VulnerabilityReportType`](#vulnerabilityreporttype) | Type of the security report that found the vulnerability finding. | | <a id="pipelinesecurityreportfindingscanner"></a>`scanner` | [`VulnerabilityScanner`](#vulnerabilityscanner) | Scanner metadata for the vulnerability. | | <a id="pipelinesecurityreportfindingseverity"></a>`severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the vulnerability finding. | @@ -14578,6 +14948,7 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="projectonlyallowmergeifalldiscussionsareresolved"></a>`onlyAllowMergeIfAllDiscussionsAreResolved` | [`Boolean`](#boolean) | Indicates if merge requests of the project can only be merged when all the discussions are resolved. | | <a id="projectonlyallowmergeifpipelinesucceeds"></a>`onlyAllowMergeIfPipelineSucceeds` | [`Boolean`](#boolean) | Indicates if merge requests of the project can only be merged with successful jobs. | | <a id="projectopenissuescount"></a>`openIssuesCount` | [`Int`](#int) | Number of open issues for the project. | +| <a id="projectpackagescleanuppolicy"></a>`packagesCleanupPolicy` | [`PackagesCleanupPolicy`](#packagescleanuppolicy) | Packages cleanup policy for the project. | | <a id="projectpath"></a>`path` | [`String!`](#string) | Path of the project. | | <a id="projectpathlocks"></a>`pathLocks` | [`PathLockConnection`](#pathlockconnection) | The project's path locks. (see [Connections](#connections)) | | <a id="projectpipelineanalytics"></a>`pipelineAnalytics` | [`PipelineAnalytics`](#pipelineanalytics) | Pipeline analytics. | @@ -14610,6 +14981,7 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="projecttopics"></a>`topics` | [`[String!]`](#string) | List of project topics. | | <a id="projectuserpermissions"></a>`userPermissions` | [`ProjectPermissions!`](#projectpermissions) | Permissions for the current user on the resource. | | <a id="projectvisibility"></a>`visibility` | [`String`](#string) | Visibility of the project. | +| <a id="projectvulnerabilityimages"></a>`vulnerabilityImages` | [`VulnerabilityContainerImageConnection`](#vulnerabilitycontainerimageconnection) | Container images reported on the project vulnerabilities. (see [Connections](#connections)) | | <a id="projectvulnerabilityscanners"></a>`vulnerabilityScanners` | [`VulnerabilityScannerConnection`](#vulnerabilityscannerconnection) | Vulnerability scanners reported on the project vulnerabilities. (see [Connections](#connections)) | | <a id="projectweburl"></a>`webUrl` | [`String`](#string) | Web URL of the project. | | <a id="projectwikienabled"></a>`wikiEnabled` | [`Boolean`](#boolean) | Indicates if Wikis are enabled for the current user. | @@ -15236,7 +15608,7 @@ Network Policies of the project. WARNING: **Deprecated** in 14.8. -Network policies are deprecated and will be removed in GitLab 16.0. +Network policies are deprecated and will be removed in GitLab 16.0. Since GitLab 15.0 this field returns no data. Returns [`NetworkPolicyConnection`](#networkpolicyconnection). @@ -15444,8 +15816,8 @@ Returns [`[SecurityTrainingUrl!]`](#securitytrainingurl). | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="projectsecuritytrainingurlsfilename"></a>`filename` | [`String`](#string) | Filename to filter security training URLs by programming language. | | <a id="projectsecuritytrainingurlsidentifierexternalids"></a>`identifierExternalIds` | [`[String!]!`](#string) | List of external IDs of vulnerability identifiers. | -| <a id="projectsecuritytrainingurlslanguage"></a>`language` | [`String`](#string) | Desired language for training urls. | ##### `Project.sentryDetailedError` @@ -15606,6 +15978,31 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="projectworkitemtypestaskable"></a>`taskable` | [`Boolean`](#boolean) | If `true`, only taskable work item types will be returned. Argument is experimental and can be removed in the future without notice. | +##### `Project.workItems` + +Work items of the project. + +WARNING: +**Introduced** in 15.1. +This feature is in Alpha. It can be changed or removed at any time. + +Returns [`WorkItemConnection`](#workitemconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectworkitemsiid"></a>`iid` | [`String`](#string) | IID of the issue. For example, "1". | +| <a id="projectworkitemsiids"></a>`iids` | [`[String!]`](#string) | List of IIDs of work items. For example, `["1", "2"]`. | +| <a id="projectworkitemssearch"></a>`search` | [`String`](#string) | Search query for title or description. | +| <a id="projectworkitemssort"></a>`sort` | [`WorkItemSort`](#workitemsort) | Sort work items by this criteria. | +| <a id="projectworkitemsstate"></a>`state` | [`IssuableState`](#issuablestate) | Current state of this work item. | +| <a id="projectworkitemstypes"></a>`types` | [`[IssueType!]`](#issuetype) | Filter work items by the given work item types. | + ### `ProjectCiCdSetting` #### Fields @@ -15809,6 +16206,7 @@ Represents a release. | <a id="releasedescription"></a>`description` | [`String`](#string) | Description (also known as "release notes") of the release. | | <a id="releasedescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | | <a id="releaseevidences"></a>`evidences` | [`ReleaseEvidenceConnection`](#releaseevidenceconnection) | Evidence for the release. (see [Connections](#connections)) | +| <a id="releaseid"></a>`id` | [`ReleaseID!`](#releaseid) | Global ID of the release. | | <a id="releaselinks"></a>`links` | [`ReleaseLinks`](#releaselinks) | Links of the release. | | <a id="releasemilestones"></a>`milestones` | [`MilestoneConnection`](#milestoneconnection) | Milestones associated to the release. (see [Connections](#connections)) | | <a id="releasename"></a>`name` | [`String`](#string) | Name of the release. | @@ -16651,6 +17049,7 @@ Completion status of tasks. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="terraformstatecreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp the Terraform state was created. | +| <a id="terraformstatedeletedat"></a>`deletedAt` | [`Time`](#time) | Timestamp the Terraform state was deleted. | | <a id="terraformstateid"></a>`id` | [`ID!`](#id) | ID of the Terraform state. | | <a id="terraformstatelatestversion"></a>`latestVersion` | [`TerraformStateVersion`](#terraformstateversion) | Latest version of the Terraform state. | | <a id="terraformstatelockedat"></a>`lockedAt` | [`Time`](#time) | Timestamp the Terraform state was locked. | @@ -16815,8 +17214,20 @@ Represents a historically accurate report about the timebox. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="timeboxreportburnuptimeseries"></a>`burnupTimeSeries` | [`[BurnupChartDailyTotals!]`](#burnupchartdailytotals) | Daily scope and completed totals for burnup charts. | +| <a id="timeboxreporterror"></a>`error` | [`TimeboxReportError`](#timeboxreporterror) | If the report cannot be generated, information about why. | | <a id="timeboxreportstats"></a>`stats` | [`TimeReportStats`](#timereportstats) | Represents the time report stats for the timebox. | +### `TimeboxReportError` + +Explains why we could not generate a timebox report. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="timeboxreporterrorcode"></a>`code` | [`TimeboxReportErrorReason`](#timeboxreporterrorreason) | Machine readable code, categorizing the error. | +| <a id="timeboxreporterrormessage"></a>`message` | [`String`](#string) | Human readable message explaining what happened. | + ### `TimelineEventType` Describes an incident management timeline event. @@ -16876,6 +17287,7 @@ Representing a to-do entry. | <a id="todocreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp this to-do item was created. | | <a id="todogroup"></a>`group` | [`Group`](#group) | Group this to-do item is associated with. | | <a id="todoid"></a>`id` | [`ID!`](#id) | ID of the to-do item. | +| <a id="todonote"></a>`note` | [`Note`](#note) | Note which created this to-do item. | | <a id="todoproject"></a>`project` | [`Project`](#project) | Project this to-do item is associated with. | | <a id="todostate"></a>`state` | [`TodoStateEnum!`](#todostateenum) | State of the to-do item. | | <a id="todotarget"></a>`target` | [`Todoable!`](#todoable) | Target of the to-do item. | @@ -17310,6 +17722,16 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="vulnerabilityissuelinkslinktype"></a>`linkType` | [`VulnerabilityIssueLinkType`](#vulnerabilityissuelinktype) | Filter issue links by link type. | +### `VulnerabilityContainerImage` + +Represents a container image reported on the related vulnerability. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="vulnerabilitycontainerimagename"></a>`name` | [`String`](#string) | Name of the container image. | + ### `VulnerabilityDetailBase` Represents the vulnerability details base. @@ -17826,6 +18248,7 @@ Represents vulnerability letter grades with associated projects. | <a id="workitemtitle"></a>`title` | [`String!`](#string) | Title of the work item. | | <a id="workitemtitlehtml"></a>`titleHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `title`. | | <a id="workitemuserpermissions"></a>`userPermissions` | [`WorkItemPermissions!`](#workitempermissions) | Permissions for the current user on the resource. | +| <a id="workitemwidgets"></a>`widgets` | [`[WorkItemWidget!]`](#workitemwidget) | Collection of widgets that belong to the work item. | | <a id="workitemworkitemtype"></a>`workItemType` | [`WorkItemType!`](#workitemtype) | Type assigned to the work item. | ### `WorkItemPermissions` @@ -17850,6 +18273,30 @@ Check permissions for the current user on a work item. | <a id="workitemtypeid"></a>`id` | [`WorkItemsTypeID!`](#workitemstypeid) | Global ID of the work item type. | | <a id="workitemtypename"></a>`name` | [`String!`](#string) | Name of the work item type. | +### `WorkItemWidgetDescription` + +Represents a description widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetdescriptiondescription"></a>`description` | [`String`](#string) | Description of the work item. | +| <a id="workitemwidgetdescriptiondescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | +| <a id="workitemwidgetdescriptiontype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + +### `WorkItemWidgetHierarchy` + +Represents a hierarchy widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgethierarchychildren"></a>`children` | [`WorkItemConnection`](#workitemconnection) | Child work items. (see [Connections](#connections)) | +| <a id="workitemwidgethierarchyparent"></a>`parent` | [`WorkItem`](#workitem) | Parent work item. | +| <a id="workitemwidgethierarchytype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ## Enumeration types Also called _Enums_, enumeration types are a special kind of scalar that @@ -18274,6 +18721,20 @@ Values for sorting tags. | <a id="containerrepositorytagsortname_asc"></a>`NAME_ASC` | Ordered by name in ascending order. | | <a id="containerrepositorytagsortname_desc"></a>`NAME_DESC` | Ordered by name in descending order. | +### `CustomerRelationsContactState` + +| Value | Description | +| ----- | ----------- | +| <a id="customerrelationscontactstateactive"></a>`active` | Active contact. | +| <a id="customerrelationscontactstateinactive"></a>`inactive` | Inactive contact. | + +### `CustomerRelationsOrganizationState` + +| Value | Description | +| ----- | ----------- | +| <a id="customerrelationsorganizationstateactive"></a>`active` | Active organization. | +| <a id="customerrelationsorganizationstateinactive"></a>`inactive` | Inactive organization. | + ### `DastProfileCadenceUnit` Unit for the duration of Dast Profile Cadence. @@ -18555,6 +19016,16 @@ Health status of an issue or epic. | <a id="healthstatusneedsattention"></a>`needsAttention` | Needs attention. | | <a id="healthstatusontrack"></a>`onTrack` | On track. | +### `IssuableResourceLinkType` + +Issuable resource link type enum. + +| Value | Description | +| ----- | ----------- | +| <a id="issuableresourcelinktypegeneral"></a>`general` | General link type. | +| <a id="issuableresourcelinktypeslack"></a>`slack` | Slack link type. | +| <a id="issuableresourcelinktypezoom"></a>`zoom` | Zoom link type. | + ### `IssuableSearchableField` Fields to perform the search in. @@ -18614,12 +19085,14 @@ Values for sorting issues. | ----- | ----------- | | <a id="issuesortblocking_issues_asc"></a>`BLOCKING_ISSUES_ASC` | Blocking issues count by ascending order. | | <a id="issuesortblocking_issues_desc"></a>`BLOCKING_ISSUES_DESC` | Blocking issues count by descending order. | +| <a id="issuesortclosed_at_asc"></a>`CLOSED_AT_ASC` | Closed time by ascending order. | +| <a id="issuesortclosed_at_desc"></a>`CLOSED_AT_DESC` | Closed time by descending order. | | <a id="issuesortcreated_asc"></a>`CREATED_ASC` | Created at ascending order. | | <a id="issuesortcreated_desc"></a>`CREATED_DESC` | Created at descending order. | | <a id="issuesortdue_date_asc"></a>`DUE_DATE_ASC` | Due date by ascending order. | | <a id="issuesortdue_date_desc"></a>`DUE_DATE_DESC` | Due date by descending order. | -| <a id="issuesortescalation_status_asc"></a>`ESCALATION_STATUS_ASC` | Status from triggered to resolved. Defaults to `CREATED_DESC` if `incident_escalations` feature flag is disabled. | -| <a id="issuesortescalation_status_desc"></a>`ESCALATION_STATUS_DESC` | Status from resolved to triggered. Defaults to `CREATED_DESC` if `incident_escalations` feature flag is disabled. | +| <a id="issuesortescalation_status_asc"></a>`ESCALATION_STATUS_ASC` | Status from triggered to resolved. | +| <a id="issuesortescalation_status_desc"></a>`ESCALATION_STATUS_DESC` | Status from resolved to triggered. | | <a id="issuesortlabel_priority_asc"></a>`LABEL_PRIORITY_ASC` | Label priority by ascending order. | | <a id="issuesortlabel_priority_desc"></a>`LABEL_PRIORITY_DESC` | Label priority by descending order. | | <a id="issuesortmilestone_due_asc"></a>`MILESTONE_DUE_ASC` | Milestone due date by ascending order. | @@ -19025,6 +19498,18 @@ Values for sorting package. | <a id="packagetypeenumrubygems"></a>`RUBYGEMS` | Packages from the Rubygems package manager. | | <a id="packagetypeenumterraform_module"></a>`TERRAFORM_MODULE` | Packages from the Terraform Module package manager. | +### `PackagesCleanupKeepDuplicatedPackageFilesEnum` + +| Value | Description | +| ----- | ----------- | +| <a id="packagescleanupkeepduplicatedpackagefilesenumall_package_files"></a>`ALL_PACKAGE_FILES` | Value to keep all package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumfifty_package_files"></a>`FIFTY_PACKAGE_FILES` | Value to keep 50 package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumforty_package_files"></a>`FORTY_PACKAGE_FILES` | Value to keep 40 package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumone_package_file"></a>`ONE_PACKAGE_FILE` | Value to keep 1 package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumten_package_files"></a>`TEN_PACKAGE_FILES` | Value to keep 10 package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumthirty_package_files"></a>`THIRTY_PACKAGE_FILES` | Value to keep 30 package files. | +| <a id="packagescleanupkeepduplicatedpackagefilesenumtwenty_package_files"></a>`TWENTY_PACKAGE_FILES` | Value to keep 20 package files. | + ### `PipelineConfigSourceEnum` | Value | Description | @@ -19039,6 +19524,16 @@ Values for sorting package. | <a id="pipelineconfigsourceenumunknown_source"></a>`UNKNOWN_SOURCE` | Unknown source. | | <a id="pipelineconfigsourceenumwebide_source"></a>`WEBIDE_SOURCE` | Webide source. | +### `PipelineMergeRequestEventType` + +Event type of the pipeline associated with a merge request. + +| Value | Description | +| ----- | ----------- | +| <a id="pipelinemergerequesteventtypedetached"></a>`DETACHED` | Pipeline run on the changes in the merge request source branch. | +| <a id="pipelinemergerequesteventtypemerged_result"></a>`MERGED_RESULT` | Pipeline run on the changes from the source branch combined with the target branch. | +| <a id="pipelinemergerequesteventtypemerge_train"></a>`MERGE_TRAIN` | Pipeline ran as part of a merge train. | + ### `PipelineScopeEnum` | Value | Description | @@ -19323,6 +19818,30 @@ State of a test report. | <a id="testreportstatefailed"></a>`FAILED` | Failed test report. | | <a id="testreportstatepassed"></a>`PASSED` | Passed test report. | +### `TimeboxReportErrorReason` + +Category of error. + +| Value | Description | +| ----- | ----------- | +| <a id="timeboxreporterrorreasoncreated_asc"></a>`CREATED_ASC` | Created at ascending order. | +| <a id="timeboxreporterrorreasoncreated_desc"></a>`CREATED_DESC` | Created at descending order. | +| <a id="timeboxreporterrorreasonlabel_priority_asc"></a>`LABEL_PRIORITY_ASC` | Label priority by ascending order. | +| <a id="timeboxreporterrorreasonlabel_priority_desc"></a>`LABEL_PRIORITY_DESC` | Label priority by descending order. | +| <a id="timeboxreporterrorreasonmilestone_due_asc"></a>`MILESTONE_DUE_ASC` | Milestone due date by ascending order. | +| <a id="timeboxreporterrorreasonmilestone_due_desc"></a>`MILESTONE_DUE_DESC` | Milestone due date by descending order. | +| <a id="timeboxreporterrorreasonmissing_dates"></a>`MISSING_DATES` | One or both of start_date and due_date is missing. | +| <a id="timeboxreporterrorreasonpriority_asc"></a>`PRIORITY_ASC` | Priority by ascending order. | +| <a id="timeboxreporterrorreasonpriority_desc"></a>`PRIORITY_DESC` | Priority by descending order. | +| <a id="timeboxreporterrorreasontoo_many_events"></a>`TOO_MANY_EVENTS` | There are too many events. | +| <a id="timeboxreporterrorreasonunsupported"></a>`UNSUPPORTED` | This type does not support timebox reports. | +| <a id="timeboxreporterrorreasonupdated_asc"></a>`UPDATED_ASC` | Updated at ascending order. | +| <a id="timeboxreporterrorreasonupdated_desc"></a>`UPDATED_DESC` | Updated at descending order. | +| <a id="timeboxreporterrorreasoncreated_asc"></a>`created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| <a id="timeboxreporterrorreasoncreated_desc"></a>`created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| <a id="timeboxreporterrorreasonupdated_asc"></a>`updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| <a id="timeboxreporterrorreasonupdated_desc"></a>`updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | + ### `TodoActionEnum` | Value | Description | @@ -19395,6 +19914,7 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumminute_limit_banner"></a>`MINUTE_LIMIT_BANNER` | Callout feature name for minute_limit_banner. | | <a id="usercalloutfeaturenameenumnew_user_signups_cap_reached"></a>`NEW_USER_SIGNUPS_CAP_REACHED` | Callout feature name for new_user_signups_cap_reached. | | <a id="usercalloutfeaturenameenumpersonal_access_token_expiry"></a>`PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for personal_access_token_expiry. | +| <a id="usercalloutfeaturenameenumpersonal_project_limitations_banner"></a>`PERSONAL_PROJECT_LIMITATIONS_BANNER` | Callout feature name for personal_project_limitations_banner. | | <a id="usercalloutfeaturenameenumpipeline_needs_banner"></a>`PIPELINE_NEEDS_BANNER` | Callout feature name for pipeline_needs_banner. | | <a id="usercalloutfeaturenameenumpipeline_needs_hover_tip"></a>`PIPELINE_NEEDS_HOVER_TIP` | Callout feature name for pipeline_needs_hover_tip. | | <a id="usercalloutfeaturenameenumpreview_user_over_limit_free_plan_alert"></a>`PREVIEW_USER_OVER_LIMIT_FREE_PLAN_ALERT` | Callout feature name for preview_user_over_limit_free_plan_alert. | @@ -19408,6 +19928,7 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumstorage_enforcement_banner_fourth_enforcement_threshold"></a>`STORAGE_ENFORCEMENT_BANNER_FOURTH_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_fourth_enforcement_threshold. | | <a id="usercalloutfeaturenameenumstorage_enforcement_banner_second_enforcement_threshold"></a>`STORAGE_ENFORCEMENT_BANNER_SECOND_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_second_enforcement_threshold. | | <a id="usercalloutfeaturenameenumstorage_enforcement_banner_third_enforcement_threshold"></a>`STORAGE_ENFORCEMENT_BANNER_THIRD_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_third_enforcement_threshold. | +| <a id="usercalloutfeaturenameenumsubmit_license_usage_data_banner"></a>`SUBMIT_LICENSE_USAGE_DATA_BANNER` | Callout feature name for submit_license_usage_data_banner. | | <a id="usercalloutfeaturenameenumsuggest_pipeline"></a>`SUGGEST_PIPELINE` | Callout feature name for suggest_pipeline. | | <a id="usercalloutfeaturenameenumsuggest_popover_dismissed"></a>`SUGGEST_POPOVER_DISMISSED` | Callout feature name for suggest_popover_dismissed. | | <a id="usercalloutfeaturenameenumtabs_position_highlight"></a>`TABS_POSITION_HIGHLIGHT` | Callout feature name for tabs_position_highlight. | @@ -19576,6 +20097,23 @@ Weight ID wildcard values. | <a id="weightwildcardidany"></a>`ANY` | Weight is assigned. | | <a id="weightwildcardidnone"></a>`NONE` | No weight is assigned. | +### `WorkItemSort` + +Values for sorting work items. + +| Value | Description | +| ----- | ----------- | +| <a id="workitemsortcreated_asc"></a>`CREATED_ASC` | Created at ascending order. | +| <a id="workitemsortcreated_desc"></a>`CREATED_DESC` | Created at descending order. | +| <a id="workitemsorttitle_asc"></a>`TITLE_ASC` | Title by ascending order. | +| <a id="workitemsorttitle_desc"></a>`TITLE_DESC` | Title by descending order. | +| <a id="workitemsortupdated_asc"></a>`UPDATED_ASC` | Updated at ascending order. | +| <a id="workitemsortupdated_desc"></a>`UPDATED_DESC` | Updated at descending order. | +| <a id="workitemsortcreated_asc"></a>`created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| <a id="workitemsortcreated_desc"></a>`created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| <a id="workitemsortupdated_asc"></a>`updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| <a id="workitemsortupdated_desc"></a>`updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | + ### `WorkItemState` State of a GitLab work item. @@ -19594,6 +20132,15 @@ Values for work item state events. | <a id="workitemstateeventclose"></a>`CLOSE` | Closes the work item. | | <a id="workitemstateeventreopen"></a>`REOPEN` | Reopens the work item. | +### `WorkItemWidgetType` + +Type of a work item widget. + +| Value | Description | +| ----- | ----------- | +| <a id="workitemwidgettypedescription"></a>`DESCRIPTION` | Description widget. | +| <a id="workitemwidgettypehierarchy"></a>`HIERARCHY` | Hierarchy widget. | + ## Scalar types Scalar values are atomic values, and do not have fields of their own. @@ -19629,6 +20176,12 @@ A `AuditEventsExternalAuditEventDestinationID` is a global ID. It is encoded as An example `AuditEventsExternalAuditEventDestinationID` is: `"gid://gitlab/AuditEvents::ExternalAuditEventDestination/1"`. +### `AuditEventsStreamingHeaderID` + +A `AuditEventsStreamingHeaderID` is a global ID. It is encoded as a string. + +An example `AuditEventsStreamingHeaderID` is: `"gid://gitlab/AuditEvents::Streaming::Header/1"`. + ### `AwardableID` A `AwardableID` is a global ID. It is encoded as a string. @@ -19878,6 +20431,12 @@ A `IncidentManagementEscalationRuleID` is a global ID. It is encoded as a string An example `IncidentManagementEscalationRuleID` is: `"gid://gitlab/IncidentManagement::EscalationRule/1"`. +### `IncidentManagementIssuableResourceLinkID` + +A `IncidentManagementIssuableResourceLinkID` is a global ID. It is encoded as a string. + +An example `IncidentManagementIssuableResourceLinkID` is: `"gid://gitlab/IncidentManagement::IssuableResourceLink/1"`. + ### `IncidentManagementOncallParticipantID` A `IncidentManagementOncallParticipantID` is a global ID. It is encoded as a string. @@ -20070,6 +20629,12 @@ A `ProjectID` is a global ID. It is encoded as a string. An example `ProjectID` is: `"gid://gitlab/Project/1"`. +### `ReleaseID` + +A `ReleaseID` is a global ID. It is encoded as a string. + +An example `ReleaseID` is: `"gid://gitlab/Release/1"`. + ### `ReleasesLinkID` A `ReleasesLinkID` is a global ID. It is encoded as a string. @@ -20789,6 +21354,19 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="usertodosstate"></a>`state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | <a id="usertodostype"></a>`type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +#### `WorkItemWidget` + +Implementations: + +- [`WorkItemWidgetDescription`](#workitemwidgetdescription) +- [`WorkItemWidgetHierarchy`](#workitemwidgethierarchy) + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgettype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ## Input types Types that may be used as arguments (all scalar types may also @@ -21260,3 +21838,21 @@ A time-frame defined as a closed inclusive range of two dates. | <a id="workitemdeletedtaskinputid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the task referenced in the work item's description. | | <a id="workitemdeletedtaskinputlinenumberend"></a>`lineNumberEnd` | [`Int!`](#int) | Last line in the Markdown source that defines the list item task. | | <a id="workitemdeletedtaskinputlinenumberstart"></a>`lineNumberStart` | [`Int!`](#int) | First line in the Markdown source that defines the list item task. | + +### `WorkItemUpdatedTaskInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemupdatedtaskinputid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| <a id="workitemupdatedtaskinputstateevent"></a>`stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | +| <a id="workitemupdatedtaskinputtitle"></a>`title` | [`String`](#string) | Title of the work item. | + +### `WorkItemWidgetDescriptionInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetdescriptioninputdescription"></a>`description` | [`String!`](#string) | Description of the work item. | diff --git a/doc/api/graphql/sample_issue_boards.md b/doc/api/graphql/sample_issue_boards.md index 68c12aa26a2..7fe8046b27d 100644 --- a/doc/api/graphql/sample_issue_boards.md +++ b/doc/api/graphql/sample_issue_boards.md @@ -37,7 +37,7 @@ instance of the [GraphiQL explorer](https://gitlab.com/-/graphql-explorer): 1. Open the [GraphiQL Explorer](https://gitlab.com/-/graphql-explorer) page. 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. -1. Click Play to get the result shown here: +1. Select Play to get the result shown here:  diff --git a/doc/api/graphql/users_example.md b/doc/api/graphql/users_example.md index 0658a9402e7..60ca7090aac 100644 --- a/doc/api/graphql/users_example.md +++ b/doc/api/graphql/users_example.md @@ -73,7 +73,7 @@ explorer. GraphiQL explorer is available for: 1. Open the [GraphiQL explorer tool](https://gitlab.com/-/graphql-explorer). 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. -1. Click Play to get the result shown here: +1. Select **Play** to get the result shown here:  diff --git a/doc/api/group_access_tokens.md b/doc/api/group_access_tokens.md index 0d1878ebf39..1c707f92ebd 100644 --- a/doc/api/group_access_tokens.md +++ b/doc/api/group_access_tokens.md @@ -83,7 +83,8 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77236) in GitLab 14.7. -Create a [group access token](../user/group/settings/group_access_tokens.md). +Create a [group access token](../user/group/settings/group_access_tokens.md). You must have the Owner role for the +group to create group access tokens. ```plaintext POST groups/:id/access_tokens @@ -94,7 +95,7 @@ POST groups/:id/access_tokens | `id` | integer or string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | | `name` | String | yes | Name of the group access token | | `scopes` | `Array[String]` | yes | [List of scopes](../user/group/settings/group_access_tokens.md#scopes-for-a-group-access-token) | -| `access_level` | Integer | no | A valid access level. Default value is 40 (Maintainer). Other allowed values are 10 (Guest), 20 (Reporter), and 30 (Developer). | +| `access_level` | Integer | no | Access level. Valid values are `10` (Guest), `20` (Reporter), `30` (Developer), `40` (Maintainer), and `50` (Owner). | | `expires_at` | Date | no | Token expires at midnight UTC on that date | ```shell diff --git a/doc/api/group_protected_environments.md b/doc/api/group_protected_environments.md index f8f9b853354..8ebd0dcd99a 100644 --- a/doc/api/group_protected_environments.md +++ b/doc/api/group_protected_environments.md @@ -28,7 +28,7 @@ Currently, these levels are recognized: Gets a list of protected environments from a group. -```shell +```plaintext GET /groups/:id/protected_environments ``` @@ -63,7 +63,7 @@ Example response: Gets a single protected environment. -```shell +```plaintext GET /groups/:id/protected_environments/:name ``` @@ -97,7 +97,7 @@ Example response: Protects a single environment. -```shell +```plaintext POST /groups/:id/protected_environments ``` @@ -137,7 +137,7 @@ Example response: Unprotects the given protected environment. -```shell +```plaintext DELETE /groups/:id/protected_environments/:name ``` diff --git a/doc/api/group_repository_storage_moves.md b/doc/api/group_repository_storage_moves.md index 1d6e08b5840..1b3940479bb 100644 --- a/doc/api/group_repository_storage_moves.md +++ b/doc/api/group_repository_storage_moves.md @@ -10,7 +10,7 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53016) in GitLab 13.9. Group repositories can be moved between storages. This API can help you when -[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrating-to-gitaly-cluster), for +[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), for example, or to migrate a [group wiki](../user/project/wiki/group.md). As group repository storage moves are processed, they transition through different states. Values diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index a78f989a755..50bb67b8173 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -72,7 +72,7 @@ GET /groups/:id/wikis/:slug | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | | `slug` | string | yes | URL-encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | | `render_html` | boolean | no | Return the rendered HTML of the wiki page | -| `version` | string | no | Wiki page version sha | +| `version` | string | no | Wiki page version SHA | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/wikis/home" diff --git a/doc/api/groups.md b/doc/api/groups.md index b565e714285..8372d6deddd 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -389,9 +389,9 @@ Example response: "tag_list":[], //deprecated, use `topics` instead "topics":[], "ssh_url_to_repo":"ssh://git@gitlab.com/h5bp/html5-boilerplate.git", - "http_url_to_repo":"http://gitlab.com/h5bp/html5-boilerplate.git", - "web_url":"http://gitlab.com/h5bp/html5-boilerplate", - "readme_url":"http://gitlab.com/h5bp/html5-boilerplate/-/blob/master/README.md", + "http_url_to_repo":"https://gitlab.com/h5bp/html5-boilerplate.git", + "web_url":"https://gitlab.com/h5bp/html5-boilerplate", + "readme_url":"https://gitlab.com/h5bp/html5-boilerplate/-/blob/master/README.md", "avatar_url":null, "star_count":0, "forks_count":4, @@ -404,16 +404,16 @@ Example response: "full_path":"h5bp", "parent_id":null, "avatar_url":null, - "web_url":"http://gitlab.com/groups/h5bp" + "web_url":"https://gitlab.com/groups/h5bp" }, "_links":{ - "self":"http://gitlab.com/api/v4/projects/8", - "issues":"http://gitlab.com/api/v4/projects/8/issues", - "merge_requests":"http://gitlab.com/api/v4/projects/8/merge_requests", - "repo_branches":"http://gitlab.com/api/v4/projects/8/repository/branches", - "labels":"http://gitlab.com/api/v4/projects/8/labels", - "events":"http://gitlab.com/api/v4/projects/8/events", - "members":"http://gitlab.com/api/v4/projects/8/members" + "self":"https://gitlab.com/api/v4/projects/8", + "issues":"https://gitlab.com/api/v4/projects/8/issues", + "merge_requests":"https://gitlab.com/api/v4/projects/8/merge_requests", + "repo_branches":"https://gitlab.com/api/v4/projects/8/repository/branches", + "labels":"https://gitlab.com/api/v4/projects/8/labels", + "events":"https://gitlab.com/api/v4/projects/8/events", + "members":"https://gitlab.com/api/v4/projects/8/members" }, "empty_repo":false, "archived":false, @@ -1060,7 +1060,7 @@ Only available to group owners and administrators. This endpoint either: - Removes group, and queues a background job to delete all projects in the group as well. -- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion happens 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +- Since [GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [Premium](https://about.gitlab.com/pricing/) or higher tiers, marks a group for deletion. The deletion happens 7 days later by default, but this can be changed in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). ```plaintext DELETE /groups/:id diff --git a/doc/api/index.md b/doc/api/index.md index 78e5f980679..c29e43d3f8e 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -90,7 +90,7 @@ end of the API URL. NOTE: In the example above, replace `gitlab.example.com` with `gitlab.com` to query GitLab.com (GitLab SaaS). -Access can be denied due to authentication. For more information, see [Authentication](#authentication). +Access can be denied due to authentication. For more information, see [Authentication](#authentication). ### API request to expose HTTP response headers @@ -127,6 +127,7 @@ There are several ways you can authenticate with the GitLab API: - [OAuth2 tokens](#oauth2-tokens) - [Personal access tokens](../user/profile/personal_access_tokens.md) - [Project access tokens](../user/project/settings/project_access_tokens.md) +- [Group access tokens](../user/group/settings/group_access_tokens.md) - [Session cookie](#session-cookie) - [GitLab CI/CD job token](../ci/jobs/ci_job_token.md) **(Specific endpoints only)** @@ -342,13 +343,14 @@ The following table gives an overview of how the API functions generally behave. | `GET` | Access one or more resources and return the result as JSON. | | `POST` | Return `201 Created` if the resource is successfully created and return the newly created resource as JSON. | | `GET` / `PUT` | Return `200 OK` if the resource is accessed or modified successfully. The (modified) result is returned as JSON. | -| `DELETE` | Returns `204 No Content` if the resource was deleted successfully. | +| `DELETE` | Returns `204 No Content` if the resource was deleted successfully or `202 Accepted` if the resource is scheduled to be deleted. | The following table shows the possible return codes for API requests. | Return values | Description | |--------------------------|-------------| | `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, and the resource itself is returned as JSON. | +| `202 Accepted` | The `GET`, `PUT` or `DELETE` request was successful, and the resource is scheduled for processing. | | `204 No Content` | The server has successfully fulfilled the request, and there is no additional content to send in the response payload body. | | `201 Created` | The `POST` request was successful, and the resource is returned as JSON. | | `304 Not Modified` | The resource hasn't been modified since the last request. | @@ -390,6 +392,9 @@ In the following example, we list 50 [namespaces](namespaces.md) per page: curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/namespaces?per_page=50" ``` +NOTE: +There is a [max offset allowed limit](../administration/instance_limits.md#max-offset-allowed-by-the-rest-api-for-offset-based-pagination) for offset pagination. You can change the limit in self-managed instances. + #### Pagination `Link` header [`Link` headers](https://www.w3.org/wiki/LinkHeader) are returned with each @@ -738,7 +743,7 @@ Content-Type: application/json ## Encoding `+` in ISO 8601 dates If you need to include a `+` in a query parameter, you may need to use `%2B` -instead, due to a [W3 recommendation](http://www.w3.org/Addressing/URL/4_URI_Recommentations.html) +instead, due to a [W3 recommendation](https://www.w3.org/Addressing/URL/4_URI_Recommentations.html) that causes a `+` to be interpreted as a space. For example, in an ISO 8601 date, you may want to include a specific time in ISO 8601 format, such as: diff --git a/doc/api/instance_clusters.md b/doc/api/instance_clusters.md index 137e8e3f25c..3d2c35a0ab0 100644 --- a/doc/api/instance_clusters.md +++ b/doc/api/instance_clusters.md @@ -221,7 +221,7 @@ Example response: Updates an existing instance cluster. -```shell +```plaintext PUT /admin/clusters/:cluster_id ``` diff --git a/doc/api/invitations.md b/doc/api/invitations.md index eb7351c2e09..5a39a86d039 100644 --- a/doc/api/invitations.md +++ b/doc/api/invitations.md @@ -22,9 +22,9 @@ levels are defined in the `Gitlab::Access` module. Currently, these levels are v - Maintainer (`40`) - Owner (`50`) - Only valid to set for groups -WARNING: -Due to [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299), -projects in personal namespaces don't show owner (`50`) permission. +NOTE: +From [GitLab 14.9](https://gitlab.com/gitlab-org/gitlab/-/issues/351211) and later, projects have a maximum role of Owner. +Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299) in GitLab 14.8 and earlier, projects have a maximum role of Maintainer. ## Add a member to a group or project diff --git a/doc/api/issue_links.md b/doc/api/issue_links.md index ac26d2a3d17..eb9e8e8adc0 100644 --- a/doc/api/issue_links.md +++ b/doc/api/issue_links.md @@ -63,6 +63,106 @@ Parameters: ] ``` +## Get an issue link + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88228) in GitLab 15.1. + +Gets details about an issue link. + +```plaintext +GET /projects/:id/issues/:issue_iid/links/:issue_link_id +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|-----------------|----------------|------------------------|-----------------------------------------------------------------------------| +| `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | +| `issue_iid` | integer | **{check-circle}** Yes | Internal ID of a project's issue. | +| `issue_link_id` | integer/string | **{check-circle}** Yes | ID of an issue relationship. | + +Response body attributes: + +| Attribute | Type | Description | +|:---------------|:-------|:------------------------------------------------------------------------------------------| +| `source_issue` | object | Details of the source issue of the relationship. | +| `target_issue` | object | Details of the target issue of the relationship. | +| `link_type` | string | Type of the relationship. Possible values are `relates_to`, `blocks` and `is_blocked_by`. | + +Example request: + +```shell +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/84/issues/14/links/1" +``` + +Example response: + +```json +{ + "source_issue" : { + "id" : 83, + "iid" : 11, + "project_id" : 4, + "created_at" : "2016-01-07T12:44:33.959Z", + "title" : "Issues with auth", + "state" : "opened", + "assignees" : [], + "assignee" : null, + "labels" : [ + "bug" + ], + "author" : { + "name" : "Alexandra Bashirian", + "avatar_url" : null, + "state" : "active", + "web_url" : "https://gitlab.example.com/eileen.lowe", + "id" : 18, + "username" : "eileen.lowe" + }, + "description" : null, + "updated_at" : "2016-01-07T12:44:33.959Z", + "milestone" : null, + "subscribed" : true, + "user_notes_count": 0, + "due_date": null, + "web_url": "http://example.com/example/example/issues/11", + "confidential": false, + "weight": null + }, + "target_issue" : { + "id" : 84, + "iid" : 14, + "project_id" : 4, + "created_at" : "2016-01-07T12:44:33.959Z", + "title" : "Issues with auth", + "state" : "opened", + "assignees" : [], + "assignee" : null, + "labels" : [ + "bug" + ], + "author" : { + "name" : "Alexandra Bashirian", + "avatar_url" : null, + "state" : "active", + "web_url" : "https://gitlab.example.com/eileen.lowe", + "id" : 18, + "username" : "eileen.lowe" + }, + "description" : null, + "updated_at" : "2016-01-07T12:44:33.959Z", + "milestone" : null, + "subscribed" : true, + "user_notes_count": 0, + "due_date": null, + "web_url": "http://example.com/example/example/issues/14", + "confidential": false, + "weight": null + }, + "link_type": "relates_to" +} +``` + ## Create an issue link Creates a two-way relation between two issues. The user must be allowed to diff --git a/doc/api/issues.md b/doc/api/issues.md index 44b947f14dc..1f5f4b4c8ae 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -170,7 +170,8 @@ Example response: "self":"http://gitlab.example.com/api/v4/projects/1/issues/76", "notes":"http://gitlab.example.com/api/v4/projects/1/issues/76/notes", "award_emoji":"http://gitlab.example.com/api/v4/projects/1/issues/76/award_emoji", - "project":"http://gitlab.example.com/api/v4/projects/1" + "project":"http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -227,7 +228,7 @@ Issues created by users on GitLab Premium or higher include the `iteration` prop "updated_at":"2022-03-14T05:21:11.929Z", "start_date":"2022-03-08", "due_date":"2022-03-14", - "web_url":"http://gitlab.com/groups/my-group/-/iterations/90" + "web_url":"https://gitlab.com/groups/my-group/-/iterations/90" } ... } @@ -396,7 +397,8 @@ Example response: "self":"http://gitlab.example.com/api/v4/projects/4/issues/41", "notes":"http://gitlab.example.com/api/v4/projects/4/issues/41/notes", "award_emoji":"http://gitlab.example.com/api/v4/projects/4/issues/41/award_emoji", - "project":"http://gitlab.example.com/api/v4/projects/4" + "project":"http://gitlab.example.com/api/v4/projects/4", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -605,7 +607,8 @@ Example response: "self":"http://gitlab.example.com/api/v4/projects/4/issues/41", "notes":"http://gitlab.example.com/api/v4/projects/4/issues/41/notes", "award_emoji":"http://gitlab.example.com/api/v4/projects/4/issues/41/award_emoji", - "project":"http://gitlab.example.com/api/v4/projects/4" + "project":"http://gitlab.example.com/api/v4/projects/4", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -769,7 +772,8 @@ Example response: "self": "http://gitlab.example:3000/api/v4/projects/1/issues/1", "notes": "http://gitlab.example:3000/api/v4/projects/1/issues/1/notes", "award_emoji": "http://gitlab.example:3000/api/v4/projects/1/issues/1/award_emoji", - "project": "http://gitlab.example:3000/api/v4/projects/1" + "project": "http://gitlab.example:3000/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "moved_to_id": null, "service_desk_reply_to": "service.desk@gitlab.com" @@ -926,7 +930,8 @@ Example response: "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://gitlab.example.com/api/v4/projects/1" + "project": "http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -1074,7 +1079,8 @@ Example response: "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://gitlab.example.com/api/v4/projects/1" + "project": "http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -1249,7 +1255,9 @@ Example response: "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://gitlab.example.com/api/v4/projects/1" + "project": "http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" + }, "task_completion_status":{ "count":0, @@ -1433,7 +1441,8 @@ Example response: "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://gitlab.example.com/api/v4/projects/1" + "project": "http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, @@ -1586,7 +1595,8 @@ Example response: "self":"https://gitlab.example.com/api/v4/projects/143/issues/1", "notes":"https://gitlab.example.com/api/v4/projects/143/issues/1/notes", "award_emoji":"https://gitlab.example.com/api/v4/projects/143/issues/1/award_emoji", - "project":"https://gitlab.example.com/api/v4/projects/143" + "project":"https://gitlab.example.com/api/v4/projects/143", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "references":{ "short":"#1", @@ -1683,7 +1693,8 @@ Example response: "self": "http://gitlab.example.com/api/v4/projects/1/issues/2", "notes": "http://gitlab.example.com/api/v4/projects/1/issues/2/notes", "award_emoji": "http://gitlab.example.com/api/v4/projects/1/issues/2/award_emoji", - "project": "http://gitlab.example.com/api/v4/projects/1" + "project": "http://gitlab.example.com/api/v4/projects/1", + "closed_as_duplicate_of": "http://gitlab.example.com/api/v4/projects/1/issues/75" }, "task_completion_status":{ "count":0, diff --git a/doc/api/license.md b/doc/api/license.md index b69a5e81ea6..eb576117d4a 100644 --- a/doc/api/license.md +++ b/doc/api/license.md @@ -1,6 +1,6 @@ --- -stage: Growth -group: Activation +stage: Fulfillment +group: Utilization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/lint.md b/doc/api/lint.md index c0d0b69dc77..d5aa6af0e34 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -126,7 +126,7 @@ Example response: "stage":"test", "before_script":[], "script":["echo 1"], - "after_script":[], + "after_script":[], "tag_list":[], "environment":null, "when":"on_success", diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index 31f1cb41eb3..1a9be725b88 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -1,6 +1,6 @@ --- -stage: Growth -group: Activation +stage: Fulfillment +group: Utilization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index abe9cb65f95..eb4b7a3dd7e 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -69,7 +69,7 @@ Parameters: | `in` | string | no | Modify the scope of the `search` attribute. `title`, `description`, or a string joining them with comma. Default is `title,description`. | | `wip` | string | no | Filter merge requests against their `wip` status. `yes` to return *only* draft merge requests, `no` to return *non-draft* merge requests. | | `not` | Hash | no | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | -| `environment` | string | no | Returns merge requests deployed to the given environment. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | +| `environment` | string | no | Returns merge requests deployed to the given environment. | | `deployed_before` | datetime | no | Return merge requests deployed before the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | | `deployed_after` | datetime | no | Return merge requests deployed after the given date/time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | @@ -265,13 +265,13 @@ Parameters: | `approved_by_ids` **(PREMIUM)** | integer array | no | Returns merge requests which have been approved by all the users with the given `id`s (Max: 5). `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | | `reviewer_id` | integer | no | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given user `id`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_username`. | | `reviewer_username` | string | no | Returns merge requests which have the user as a [reviewer](../user/project/merge_requests/getting_started.md#reviewer) with the given `username`. `None` returns merge requests with no reviewers. `Any` returns merge requests with any reviewer. Mutually exclusive with `reviewer_id`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49341) in GitLab 13.8. | - | `my_reaction_emoji` | string | no | Return merge requests reacted by the authenticated user by the given `emoji`. `None` returns issues not given a reaction. `Any` returns issues given at least one reaction. | | `source_branch` | string | no | Return merge requests with the given source branch. | | `target_branch` | string | no | Return merge requests with the given target branch. | | `search` | string | no | Search merge requests against their `title` and `description`. | | `wip` | string | no | Filter merge requests against their `wip` status. `yes` to return *only* draft merge requests, `no` to return *non-draft* merge requests. | | `not` | Hash | no | Return merge requests that do not match the parameters supplied. Accepts: `labels`, `milestone`, `author_id`, `author_username`, `assignee_id`, `assignee_username`, `reviewer_id`, `reviewer_username`, `my_reaction_emoji`. | +| `environment` | string | no | Returns merge requests deployed to the given environment. | ```json [ @@ -619,92 +619,56 @@ Parameters: ```json { - "id": 1, - "iid": 1, - "project_id": 3, - "title": "test1", - "description": "fixed login page css paddings", - "state": "merged", - "created_at": "2017-04-29T08:46:00Z", - "updated_at": "2017-04-29T08:46:00Z", + "id": 155016530, + "iid": 133, + "project_id": 15513260, + "title": "Manual job rules", + "description": "", + "state": "opened", + "created_at": "2022-05-13T07:26:38.402Z", + "updated_at": "2022-05-14T03:38:31.354Z", + "merged_by": null, // Deprecated and will be removed in API v5, use `merge_user` instead + "merge_user": null, + "merged_at": null, + "closed_by": null, + "closed_at": null, "target_branch": "master", - "source_branch": "test1", + "source_branch": "manual-job-rules", + "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": { - "id": 1, - "name": "Administrator", - "username": "admin", + "id": 4155490, + "username": "marcel.amirault", + "name": "Marcel Amirault", "state": "active", - "avatar_url": null, - "web_url" : "https://gitlab.example.com/admin" - }, - "user" : { - "can_merge" : false - }, - "assignee": { - "id": 1, - "name": "Administrator", - "username": "admin", - "state": "active", - "avatar_url": null, - "web_url" : "https://gitlab.example.com/admin" - }, - "assignees": [{ - "name": "Miss Monserrate Beier", - "username": "axel.block", - "id": 12, - "state": "active", - "avatar_url": "http://www.gravatar.com/avatar/46f6f7dc858ada7be1853f7fb96e81da?s=80&d=identicon", - "web_url": "https://gitlab.example.com/axel.block" - }], - "reviewers": [{ - "id": 2, - "name": "Sam Bauch", - "username": "kenyatta_oconnell", - "state": "active", - "avatar_url": "https://www.gravatar.com/avatar/956c92487c6f6f7616b536927e22c9a0?s=80&d=identicon", - "web_url": "http://gitlab.example.com//kenyatta_oconnell" - }], - "source_project_id": 2, - "target_project_id": 3, - "labels": [ - "Community contribution", - "Manage" - ], + "avatar_url": "https://gitlab.com/uploads/-/system/user/avatar/4155490/avatar.png", + "web_url": "https://gitlab.com/marcel.amirault" + }, + "assignees": [], + "assignee": null, + "reviewers": [], + "source_project_id": 15513260, + "target_project_id": 15513260, + "labels": [], "draft": false, "work_in_progress": false, - "milestone": { - "id": 5, - "iid": 1, - "project_id": 3, - "title": "v2.0", - "description": "Assumenda aut placeat expedita exercitationem labore sunt enim earum.", - "state": "closed", - "created_at": "2015-02-02T19:49:26.013Z", - "updated_at": "2015-02-02T19:49:26.013Z", - "due_date": "2018-09-22", - "start_date": "2018-08-08", - "web_url": "https://gitlab.example.com/my-group/my-project/milestones/1" - }, - "merge_when_pipeline_succeeds": true, + "milestone": null, + "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", - "merge_error": null, - "sha": "8888888888888888888888888888888888888888", + "sha": "e82eb4a098e32c796079ca3915e07487fc4db24c", "merge_commit_sha": null, "squash_commit_sha": null, - "user_notes_count": 1, "discussion_locked": null, - "should_remove_source_branch": true, - "force_remove_source_branch": false, - "allow_collaboration": false, - "allow_maintainer_to_push": false, - "web_url": "http://gitlab.example.com/my-group/my-project/merge_requests/1", + "should_remove_source_branch": null, + "force_remove_source_branch": true, + "reference": "!133", "references": { - "short": "!1", - "relative": "!1", - "full": "my-group/my-project!1" + "short": "!133", + "relative": "!133", + "full": "marcel.amirault/test-project!133" }, + "web_url": "https://gitlab.com/marcel.amirault/test-project/-/merge_requests/133", "time_stats": { "time_estimate": 0, "total_time_spent": 0, @@ -712,51 +676,80 @@ Parameters: "human_total_time_spent": null }, "squash": false, - "subscribed": false, - "changes_count": "1", - "merged_by": { // Deprecated and will be removed in API v5, use `merge_user` instead - "id": 87854, - "name": "Douwe Maan", - "username": "DouweM", - "state": "active", - "avatar_url": "https://gitlab.example.com/uploads/-/system/user/avatar/87854/avatar.png", - "web_url": "https://gitlab.com/DouweM" - }, - "merge_user": { - "id": 87854, - "name": "Douwe Maan", - "username": "DouweM", - "state": "active", - "avatar_url": "https://gitlab.example.com/uploads/-/system/user/avatar/87854/avatar.png", - "web_url": "https://gitlab.com/DouweM" + "task_completion_status": { + "count": 0, + "completed_count": 0 }, - "merged_at": "2018-09-07T11:16:17.520Z", - "closed_by": null, - "closed_at": null, - "latest_build_started_at": "2018-09-07T07:27:38.472Z", - "latest_build_finished_at": "2018-09-07T08:07:06.012Z", + "has_conflicts": false, + "blocking_discussions_resolved": true, + "approvals_before_merge": null, + "subscribed": true, + "changes_count": "1", + "latest_build_started_at": "2022-05-13T09:46:50.032Z", + "latest_build_finished_at": null, "first_deployed_to_production_at": null, - "pipeline": { - "id": 29626725, - "sha": "2be7ddb704c7b6b83732fdd5b9f09d5a397b5f8f", - "ref": "patch-28", - "status": "success", - "web_url": "https://gitlab.example.com/my-group/my-project/pipelines/29626725" + "pipeline": { // Old parameter, use `head_pipeline` instead. + "id": 538317940, + "iid": 1877, + "project_id": 15513260, + "sha": "1604b0c46c395822e4e9478777f8e54ac99fe5b9", + "ref": "refs/merge-requests/133/merge", + "status": "failed", + "source": "merge_request_event", + "created_at": "2022-05-13T09:46:39.560Z", + "updated_at": "2022-05-13T09:47:20.706Z", + "web_url": "https://gitlab.com/marcel.amirault/test-project/-/pipelines/538317940" + }, + "head_pipeline": { + "id": 538317940, + "iid": 1877, + "project_id": 15513260, + "sha": "1604b0c46c395822e4e9478777f8e54ac99fe5b9", + "ref": "refs/merge-requests/133/merge", + "status": "failed", + "source": "merge_request_event", + "created_at": "2022-05-13T09:46:39.560Z", + "updated_at": "2022-05-13T09:47:20.706Z", + "web_url": "https://gitlab.com/marcel.amirault/test-project/-/pipelines/538317940", + "before_sha": "1604b0c46c395822e4e9478777f8e54ac99fe5b9", + "tag": false, + "yaml_errors": null, + "user": { + "id": 4155490, + "username": "marcel.amirault", + "name": "Marcel Amirault", + "state": "active", + "avatar_url": "https://gitlab.com/uploads/-/system/user/avatar/4155490/avatar.png", + "web_url": "https://gitlab.com/marcel.amirault" + }, + "started_at": "2022-05-13T09:46:50.032Z", + "finished_at": "2022-05-13T09:47:20.697Z", + "committed_at": null, + "duration": 30, + "queued_duration": 10, + "coverage": null, + "detailed_status": { + "icon": "status_failed", + "text": "failed", + "label": "failed", + "group": "failed", + "tooltip": "failed", + "has_details": true, + "details_path": "/marcel.amirault/test-project/-/pipelines/538317940", + "illustration": null, + "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png" + } }, "diff_refs": { - "base_sha": "c380d3acebd181f13629a25d2e2acca46ffe1e00", - "head_sha": "2be7ddb704c7b6b83732fdd5b9f09d5a397b5f8f", - "start_sha": "c380d3acebd181f13629a25d2e2acca46ffe1e00" + "base_sha": "1162f719d711319a2efb2a35566f3bfdadee8bab", + "head_sha": "e82eb4a098e32c796079ca3915e07487fc4db24c", + "start_sha": "1162f719d711319a2efb2a35566f3bfdadee8bab" }, - "diverged_commits_count": 2, - "rebase_in_progress": false, + "merge_error": null, "first_contribution": false, - "task_completion_status":{ - "count":0, - "completed_count":0 - }, - "has_conflicts": false, - "blocking_discussions_resolved": true + "user": { + "can_merge": true + } } ``` @@ -1629,7 +1622,7 @@ This API returns specific HTTP status codes on failure: |:------------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| | `401` | `Unauthorized` | This user does not have permission to accept this merge request. | | `405` | `Method Not Allowed` | The merge request cannot be accepted because it is `Draft`, `Closed`, `Pipeline Pending Completion`, or `Failed`. `Success` is required. | -| `406` | `Branch cannot be merged` | The branch has conflicts and cannot be merged. | +| `406` | `Branch cannot be merged` | The merge request can not be merged. | | `409` | `SHA does not match HEAD of source branch` | The provided `sha` parameter does not match the HEAD of the source. | For additional important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index 2b52166d954..1a4dfdbac2c 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -26,7 +26,7 @@ Read more on [pagination](index.md#pagination). Get all Merge Trains of the requested project: -```shell +```plaintext GET /projects/:id/merge_trains GET /projects/:id/merge_trains?scope=complete ``` diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index aa9a86f33d5..be58d75333d 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -15,6 +15,16 @@ To configure GitLab for this, see This functionality is based on the [doorkeeper Ruby gem](https://github.com/doorkeeper-gem/doorkeeper). +## CORS preflight requests + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364680) in GitLab 15.1. + +The following endpoints support [CORS preflight requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS): + +- `/oauth/revoke` +- `/oauth/token` +- `/oauth/userinfo` + ## Supported OAuth 2.0 flows GitLab supports the following authorization flows: diff --git a/doc/api/packages/conan.md b/doc/api/packages/conan.md index f5d08ed7ef8..546a472ea53 100644 --- a/doc/api/packages/conan.md +++ b/doc/api/packages/conan.md @@ -456,7 +456,7 @@ Download a recipe file to the package registry. You must use a download URL that [recipe download URLs endpoint](#recipe-download-urls) returned. -```shell +```plaintext GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name ``` @@ -489,7 +489,7 @@ Upload a recipe file to the package registry. You must use an upload URL that th [recipe upload URLs endpoint](#recipe-upload-urls) returned. -```shell +```plaintext GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/export/:file_name ``` @@ -518,7 +518,7 @@ Download a package file to the package registry. You must use a download URL tha [package download URLs endpoint](#package-download-urls) returned. -```shell +```plaintext GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name ``` @@ -553,7 +553,7 @@ Upload a package file to the package registry. You must use an upload URL that t [package upload URLs endpoint](#package-upload-urls) returned. -```shell +```plaintext GET packages/conan/v1/files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision/package/:conan_package_reference/:package_revision/:file_name ``` diff --git a/doc/api/packages/pypi.md b/doc/api/packages/pypi.md index 5ed162a3d05..3e23ded61f4 100644 --- a/doc/api/packages/pypi.md +++ b/doc/api/packages/pypi.md @@ -50,6 +50,47 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. +## Group-level simple API index + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327595) in GitLab 15.1. + +Returns a list of packages in the group as an HTML file: + +```plaintext +GET groups/:id/-/packages/pypi/simple +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | string | yes | The ID or full path of the group. | + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple" +``` + +Example response: + +```html +<!DOCTYPE html> +<html> + <head> + <title>Links for Group</title> + </head> + <body> + <h1>Links for Group</h1> + <a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my-pypi-package" data-requires-python="">my.pypi.package</a><br><a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/package-2" data-requires-python="3.8">package_2</a><br> + </body> +</html> +``` + +To write the output to a file: + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple" >> simple_index.html +``` + +This writes the downloaded file to `simple_index.html` in the current directory. + ## Group level simple API entry point > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225545) in GitLab 13.12. @@ -57,7 +98,7 @@ directory. Returns the package descriptor as an HTML file: ```plaintext -GET groups/:id/packages/pypi/simple/:package_name +GET groups/:id/-/packages/pypi/simple/:package_name ``` | Attribute | Type | Required | Description | @@ -66,7 +107,7 @@ GET groups/:id/packages/pypi/simple/:package_name | `package_name` | string | yes | The name of the package. | ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my.pypi.package" ``` Example response: @@ -79,7 +120,7 @@ Example response: </head> <body> <h1>Links for my.pypi.package</h1> - <a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python=">=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python=">=3.6">my.pypi.package-0.0.1.tar.gz</a><br> + <a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff/my.pypi.package-0.0.1-py3-none-any.whl#sha256=5y57017232013c8ac80647f4ca153k3726f6cba62d055cd747844ed95b3c65ff" data-requires-python=">=3.6">my.pypi.package-0.0.1-py3-none-any.whl</a><br><a href="https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/files/9s9w01b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2/my.pypi.package-0.0.1.tar.gz#sha256=9s9w011b0bcd52b709ec052084e33a5517ffca96f7728ddd9f8866a30cdf76f2" data-requires-python=">=3.6">my.pypi.package-0.0.1.tar.gz</a><br> </body> </html> ``` @@ -87,7 +128,7 @@ Example response: To write the output to a file: ```shell -curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/packages/pypi/simple/my.pypi.package" >> simple.html +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/groups/1/-/packages/pypi/simple/my.pypi.package" >> simple.html ``` This writes the downloaded file to `simple.html` in the current directory. @@ -122,6 +163,47 @@ curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v This writes the downloaded file to `my.pypi.package-0.0.1.tar.gz` in the current directory. +## Project-level simple API index + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327595) in GitLab 15.1. + +Returns a list of packages in the project as an HTML file: + +```plaintext +GET projects/:id/packages/pypi/simple +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | string | yes | The ID or full path of the project. | + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple" +``` + +Example response: + +```html +<!DOCTYPE html> +<html> + <head> + <title>Links for Project</title> + </head> + <body> + <h1>Links for Project</h1> + <a href="https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple/my-pypi-package" data-requires-python="">my.pypi.package</a><br><a href="https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple/package-2" data-requires-python="3.8">package_2</a><br> + </body> +</html> +``` + +To write the output to a file: + +```shell +curl --user <username>:<personal_access_token> "https://gitlab.example.com/api/v4/projects/1/packages/pypi/simple" >> simple_index.html +``` + +This writes the downloaded file to `simple_index.html` in the current directory. + ## Project-level simple API entry point > Introduced in GitLab 12.10. diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index c49af2a745c..46a4c674560 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -70,6 +70,29 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a ] ``` +## Get single personal access token by ID + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362239) in GitLab 15.1. + +Get a single personal access token by its ID. Users can get their own tokens. +Administrators can get any token. + +```plaintext +GET /personal_access_tokens/:id +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer/string | yes | ID of personal access token | + +```shell +curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/personal_access_tokens/<id>" +``` + +### Responses + +- `401: Unauthorized` if the user doesn't have access to the token they're requesting the ID or if the token with matching ID doesn't exist. + ## Revoke a personal access token Revoke a personal access token by either: diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index 5472e5bc334..8db071bf811 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -4,13 +4,13 @@ group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Pipeline triggers API **(FREE)** +# Pipeline trigger tokens API **(FREE)** You can read more about [triggering pipelines through the API](../ci/triggers/index.md). -## List project triggers +## List project trigger tokens -Get a list of project's build triggers. +Get a list of a project's pipeline trigger tokens. ```plaintext GET /projects/:id/triggers @@ -41,9 +41,9 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a The trigger token is displayed in full if the trigger token was created by the authenticated user. Trigger tokens created by other users are shortened to four characters. -## Get trigger details +## Get trigger token details -Get details of project's build trigger. +Get details of a project's pipeline trigger. ```plaintext GET /projects/:id/triggers/:trigger_id @@ -70,9 +70,9 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a } ``` -## Create a project trigger +## Create a trigger token -Create a trigger for a project. +Create a pipeline trigger for a project. ```plaintext POST /projects/:id/triggers @@ -100,9 +100,9 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ } ``` -## Update a project trigger +## Update a project trigger token -Update a trigger for a project. +Update a pipeline trigger token for a project. ```plaintext PUT /projects/:id/triggers/:trigger_id @@ -131,9 +131,9 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>" \ } ``` -## Remove a project trigger +## Remove a project trigger token -Remove a project's build trigger. +Remove a project's pipeline trigger token. ```plaintext DELETE /projects/:id/triggers/:trigger_id @@ -147,3 +147,78 @@ DELETE /projects/:id/triggers/:trigger_id ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/triggers/5" ``` + +## Trigger a pipeline with a token + +Trigger a pipeline by using a pipeline [trigger token](../ci/triggers/index.md#create-a-trigger-token) +or a [CI/CD job token](../ci/jobs/ci_job_token.md) for authentication. + +With a CI/CD job token, the [triggered pipeline is a multi-project pipeline](../ci/jobs/ci_job_token.md#trigger-a-multi-project-pipeline-by-using-a-cicd-job-token). +The job that authenticates the request becomes associated with the upstream pipeline, +which is visible on the [pipeline graph](../ci/pipelines/multi_project_pipelines.md#multi-project-pipeline-visualization). + +If you use a trigger token in a job, the job is not associated with the upstream pipeline. + +```plaintext +POST /projects/:id/trigger/pipeline +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|:------------|:---------------|:-----------------------|:---------------------| +| `id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `ref` | string | **{check-circle}** Yes | The branch or tag to run the pipeline on. | +| `token` | string | **{check-circle}** Yes | The trigger token or CI/CD job token. | +| `variables` | array | **{dotted-circle}** No | An array containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }, {'key': 'TEST', 'value': 'test variable'}]`. If `variable_type` is excluded, it defaults to `env_var`. | + +Example request: + +```shell +curl --request POST "https://gitlab.example.com/api/v4/projects/123/trigger/pipeline?token=2cb1840fb9dfc9fb0b7b1609cd29cb&ref=main" +``` + +Example response: + +```json +{ + "id": 257, + "iid": 118, + "project_id": 21, + "sha": "91e2711a93e5d9e8dddfeb6d003b636b25bf6fc9", + "ref": "main", + "status": "created", + "source": "trigger", + "created_at": "2022-03-31T01:12:49.068Z", + "updated_at": "2022-03-31T01:12:49.068Z", + "web_url": "http://127.0.0.1:3000/test-group/test-project/-/pipelines/257", + "before_sha": "0000000000000000000000000000000000000000", + "tag": false, + "yaml_errors": null, + "user": { + "id": 1, + "username": "root", + "name": "Administrator", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "http://127.0.0.1:3000/root" + }, + "started_at": null, + "finished_at": null, + "committed_at": null, + "duration": null, + "queued_duration": null, + "coverage": null, + "detailed_status": { + "icon": "status_created", + "text": "created", + "label": "created", + "group": "created", + "tooltip": "created", + "has_details": true, + "details_path": "/test-group/test-project/-/pipelines/257", + "illustration": null, + "favicon": "/assets/ci_favicons/favicon_status_created-4b975aa976d24e5a3ea7cd9a5713e6ce2cd9afd08b910415e96675de35f64955.png" + } +} +``` diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index b05c71d2748..99009b6913d 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -165,7 +165,7 @@ Example of response > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202525) in GitLab 13.0. NOTE: -This API route is part of the [Unit test report](../ci/unit_test_reports.md) feature. +This API route is part of the [Unit test report](../ci/testing/unit_test_reports.md) feature. ```plaintext GET /projects/:id/pipelines/:pipeline_id/test_report @@ -221,7 +221,7 @@ Sample response: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65471) in GitLab 14.2. NOTE: -This API route is part of the [Unit test report](../ci/unit_test_reports.md) feature. +This API route is part of the [Unit test report](../ci/testing/unit_test_reports.md) feature. ```plaintext GET /projects/:id/pipelines/:pipeline_id/test_report_summary @@ -422,6 +422,15 @@ Response: > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22988) in GitLab 11.6. +Deleting a pipeline expires all pipeline caches, and deletes all immediately +related objects, such as builds, logs, artifacts, and triggers. +**This action cannot be undone.** + +Deleting a pipeline does not automatically delete its +[child pipelines](../ci/pipelines/parent_child_pipelines.md). +See the [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/39503) +for details. + ```plaintext DELETE /projects/:id/pipelines/:pipeline_id ``` diff --git a/doc/api/project_access_tokens.md b/doc/api/project_access_tokens.md index b13005ec436..f76795f424e 100644 --- a/doc/api/project_access_tokens.md +++ b/doc/api/project_access_tokens.md @@ -87,17 +87,25 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a Create a [project access token](../user/project/settings/project_access_tokens.md). +When you create a project access token, the maximum role (access level) you set depends on if you have the Owner or Maintainer role for the group. For example, the maximum +role that can be set is: + +- Owner (`50`), if you have the Owner role for the project. +- Maintainer (`40`), if you have the Maintainer role on the project. + +In GitLab 14.8 and earlier, project access tokens have a maximum role of Maintainer. + ```plaintext POST projects/:id/access_tokens ``` -| Attribute | Type | required | Description | -|-----------|---------|----------|---------------------| -| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) | -| `name` | String | yes | Name of the project access token | -| `scopes` | `Array[String]` | yes | [List of scopes](../user/project/settings/project_access_tokens.md#scopes-for-a-project-access-token) | -| `access_level` | Integer | no | A valid access level. Default value is 40 (Maintainer). Other allowed values are 10 (Guest), 20 (Reporter), and 30 (Developer). | -| `expires_at` | Date | no | Token expires at midnight UTC on that date | +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) | +| `name` | String | yes | Name of the project access token | +| `scopes` | `Array[String]` | yes | [List of scopes](../user/project/settings/project_access_tokens.md#scopes-for-a-project-access-token) | +| `access_level` | Integer | no | Access level. Valid values are `10` (Guest), `20` (Reporter), `30` (Developer), `40` (Maintainer), and `50` (Owner). Defaults to `40`. | +| `expires_at` | Date | no | Token expires at midnight UTC on that date | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 5525000e336..87a629372d5 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -88,7 +88,7 @@ Example response: Gets a single project cluster. -```shell +```plaintext GET /projects/:id/clusters/:cluster_id ``` @@ -182,7 +182,7 @@ Example response: Adds an existing Kubernetes cluster to the project. -```shell +```plaintext POST /projects/:id/clusters/user ``` @@ -279,7 +279,7 @@ Example response: Updates an existing project cluster. -```shell +```plaintext PUT /projects/:id/clusters/:cluster_id ``` diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 3f2cc09aa1e..635cb04e2e8 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -152,16 +152,14 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" --form "path=a --form "file=@/path/to/file" "https://gitlab.example.com/api/v4/projects/import" ``` -cURL doesn't support posting a file from a remote server. Importing a project from a remote server can be accomplished through something like the following: +cURL doesn't support posting a file from a remote server. This example imports a project +using Python's `open` method: ```python import requests -from io import BytesIO - -s3_file = requests.get(presigned_url) url = 'https://gitlab.example.com/api/v4/projects/import' -files = {'file': ('file.tar.gz', BytesIO(s3_file.content))} +files = { "file": open("project_export.tar.gz", "rb") } data = { "path": "example-project", "namespace": "example-group" @@ -293,6 +291,27 @@ curl --request POST \ }' ``` +This example imports from an Amazon S3 bucket, using a module that connects to Amazon S3: + +```python +import requests +from io import BytesIO + +s3_file = requests.get(presigned_url) + +url = 'https://gitlab.example.com/api/v4/projects/import' +files = {'file': ('file.tar.gz', BytesIO(s3_file.content))} +data = { + "path": "example-project", + "namespace": "example-group" +} +headers = { + 'Private-Token': "<your_access_token>" +} + +requests.post(url, headers=headers, data=data, files=files) +``` + ```json { "id": 1, diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index b779a0046c6..ebb15e1c1d6 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -10,7 +10,7 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31285) in GitLab 13.0. Project repositories including wiki and design repositories can be moved between storages. This can be useful when -[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrating-to-gitaly-cluster), +[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), for example. As project repository storage moves are processed, they transition through different states. Values diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index 90bd2cd6f83..0429c8abe3c 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -234,8 +234,8 @@ Parameters: |:-------------|:---------------|:---------|:----------------------------------------------------------------------------------------------------------------| | `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `snippet_id` | integer | yes | The ID of a project's snippet | -| `ref` | string | yes | The name of a branch, tag or commit e.g. master | -| `file_path` | string | yes | The URL-encoded path to the file, e.g. snippet%2Erb | +| `ref` | string | yes | The name of a branch, tag or commit, for example, main | +| `file_path` | string | yes | The URL-encoded path to the file, for example, snippet%2Erb | Example request: diff --git a/doc/api/projects.md b/doc/api/projects.md index 64a2e0d801b..3cb75c39980 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -2164,7 +2164,7 @@ This endpoint: administrators can [configure](../user/group/index.md#enable-delayed-project-deletion) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after the number of days specified in the - [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). + [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). WARNING: The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index fa099afd680..46cb461b64b 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -34,7 +34,7 @@ The following types are recognized: Gets a list of protected environments from a project: -```shell +```plaintext GET /projects/:id/protected_environments ``` @@ -70,7 +70,7 @@ Example response: Gets a single protected environment: -```shell +```plaintext GET /projects/:id/protected_environments/:name ``` @@ -105,7 +105,7 @@ Example response: Protects a single environment: -```shell +```plaintext POST /projects/:id/protected_environments ``` @@ -170,7 +170,7 @@ Example response: Unprotects the given protected environment: -```shell +```plaintext DELETE /projects/:id/protected_environments/:name ``` diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 6023adb79d0..7489d0bdc9e 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -377,6 +377,7 @@ POST /projects/:id/releases | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). | | `name` | string | no | The release name. | | `tag_name` | string | yes | The tag where the release is created from. | +| `tag_message` | string | no | Message to use if creating a new annotated tag. | | `description` | string | no | The description of the release. You can use [Markdown](../../user/markdown.md). | | `ref` | string | yes, if `tag_name` doesn't exist | If a tag specified in `tag_name` doesn't exist, the release is created from `ref` and tagged with `tag_name`. It can be a commit SHA, another tag name, or a branch name. | | `milestones` | array of string | no | The title of each milestone the release is associated with. [GitLab Premium](https://about.gitlab.com/pricing/) customers can specify group milestones. | diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index 66808900ab6..c239d06ecfd 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -6,21 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Release links API **(FREE)** +> Support for [GitLab CI/CD job token](../../ci/jobs/ci_job_token.md) authentication [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250819) in GitLab 15.1. + Use this API to manipulate GitLab [Release](../../user/project/releases/index.md) links. For manipulating other Release assets, see [Release API](index.md). GitLab supports links to `http`, `https`, and `ftp` assets. -## Authentication - -For authentication, the Release Links API accepts: - -- A [Personal Access Token](../../user/profile/personal_access_tokens.md) using the - `PRIVATE-TOKEN` header. -- A [Project Access Token](../../user/project/settings/project_access_tokens.md) using the `PRIVATE-TOKEN` header. - -The [GitLab CI/CD job token](../../ci/jobs/ci_job_token.md) `$CI_JOB_TOKEN` is not supported. See [GitLab issue #50819](https://gitlab.com/gitlab-org/gitlab/-/issues/250819) for more details. - ## Get links Get assets as links from a Release. diff --git a/doc/api/repositories.md b/doc/api/repositories.md index ec5c97e5b25..1a1eee175f7 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -17,7 +17,7 @@ This command provides essentially the same functionality as the `git ls-tree` co WARNING: This endpoint is changing to keyset-based pagination. Iterating pages of results with a number (`?page=2`) [is deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67509). -Support for iterating with a number will become unsupported in GitLab 15.0. Use +Support for iterating with a number became supported in GitLab 15.0. Use the new [keyset pagination system](index.md#keyset-based-pagination) instead. ```plaintext @@ -327,6 +327,11 @@ GitLab treats trailers case-sensitively. If you set the `trailer` field to `Example`, GitLab _won't_ include commits that use the trailer `example`, `eXaMpLE`, or anything else that isn't _exactly_ `Example`. +WARNING: +The allowed commits range between `from` and `to` is limited to 15000 commits. To disable +this restriction, [turn off the feature flag](../administration/feature_flags.md) +`changelog_commits_limitation`. + If the `from` attribute is unspecified, GitLab uses the Git tag of the last stable version that came before the version specified in the `version` attribute. This requires that Git tag names follow a specific format, allowing diff --git a/doc/api/runners.md b/doc/api/runners.md index 7519c3595b6..2b31c1cc064 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -48,7 +48,7 @@ GET /runners?tag_list=tag1,tag2 |------------|--------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `scope` | string | no | Deprecated: Use `type` or `status` instead. The scope of specific runners to show, one of: `active`, `paused`, `online` and `offline`; showing all runners if none provided | | `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | -| `status` | string | no | The status of runners to show, one of: `online`, `offline` and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | +| `status` | string | no | The status of runners to show, one of: `online`, `offline`, `stale`, and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | | `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | | `tag_list` | string array | no | List of the runner's tags | @@ -113,7 +113,7 @@ GET /runners/all?tag_list=tag1,tag2 |------------|--------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `scope` | string | no | Deprecated: Use `type` or `status` instead. The scope of runners to show, one of: `specific`, `shared`, `active`, `paused`, `online` and `offline`; showing all runners if none provided | | `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | -| `status` | string | no | The status of runners to show, one of: `online` and `offline`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | +| `status` | string | no | The status of runners to show, one of: `online`, `offline`, `stale`, and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | | `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | | `tag_list` | string array | no | List of the runner's tags | @@ -370,7 +370,7 @@ GET /runners/:id/jobs | `id` | integer | yes | The ID of a runner | | `status` | string | no | Status of the job; one of: `running`, `success`, `failed`, `canceled` | | `order_by`| string | no | Order jobs by `id`. | -| `sort` | string | no | Sort jobs in `asc` or `desc` order (default: `desc`) | +| `sort` | string | no | Sort jobs in `asc` or `desc` order (default: `desc`). Specify `order_by` as well, including for `id`. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/runners/1/jobs?status=running" @@ -464,7 +464,7 @@ GET /projects/:id/runners?tag_list=tag1,tag2 | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `scope` | string | no | Deprecated: Use `type` or `status` instead. The scope of specific runners to show, one of: `active`, `paused`, `online` and `offline`; showing all runners if none provided | | `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type` | -| `status` | string | no | The status of runners to show, one of: `online` and `offline`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | +| `status` | string | no | The status of runners to show, one of: `online`, `offline`, `stale`, and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | | `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | | `tag_list` | string array | no | List of the runner's tags | @@ -580,7 +580,7 @@ GET /groups/:id/runners?tag_list=tag1,tag2 |------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer | yes | The ID of the group owned by the authenticated user | | `type` | string | no | The type of runners to show, one of: `instance_type`, `group_type`, `project_type`. The `project_type` value is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351466) and will be removed in GitLab 15.0 | -| `status` | string | no | The status of runners to show, one of: `online` and `offline`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | +| `status` | string | no | The status of runners to show, one of: `online`, `offline`, `stale`, and `never_contacted`. `active` and `paused` are also possible values which were deprecated in GitLab 14.8 and will be removed in GitLab 16.0 | | `paused` | boolean | no | Whether to include only runners that are accepting or ignoring new jobs | | `tag_list` | string array | no | List of the runner's tags | diff --git a/doc/api/search.md b/doc/api/search.md index 8c681904e98..4e644a6c087 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -27,13 +27,13 @@ GET /search | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, users. +Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `snippet_titles`, `users`. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. -### Scope: projects +### Scope: `projects` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=projects&search=flight" @@ -65,7 +65,7 @@ Example response: ] ``` -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=issues&search=file" @@ -131,7 +131,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=merge_requests&search=file" @@ -210,7 +210,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=milestones&search=release" @@ -235,7 +235,7 @@ Example response: ] ``` -### Scope: snippet_titles +### Scope: `snippet_titles` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=snippet_titles&search=sample" @@ -266,11 +266,11 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye" @@ -301,7 +301,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/search?scope=commits&search=bye" @@ -336,7 +336,7 @@ Example response: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -377,7 +377,7 @@ NOTE: > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -450,13 +450,13 @@ GET /groups/:id/search | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: projects, issues, merge_requests, milestones, users. +Search the expression in the specified scope. These scopes are supported: `projects`, `issues`, `merge_requests`, `milestones`, `users`. -If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs, notes, and commits. Find more about [the feature](../integration/elasticsearch.md). **(PREMIUM)** +If Elasticsearch is enabled additional scopes available are `blobs`, `wiki_blobs`, `notes`, and `commits`. Find more about [the feature](../integration/advanced_search/elasticsearch.md). **(PREMIUM)** The response depends on the requested scope. -### Scope: projects +### Scope: `projects` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=projects&search=flight" @@ -488,7 +488,7 @@ Example response: ] ``` -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=issues&search=file" @@ -554,7 +554,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=merge_requests&search=file" @@ -633,7 +633,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=milestones&search=release" @@ -658,11 +658,11 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=wiki_blobs&search=bye" @@ -689,11 +689,11 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: commits **(PREMIUM)** +### Scope: `commits` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/6/search?scope=commits&search=bye" @@ -724,11 +724,11 @@ Example response: ] ``` -### Scope: blobs **(PREMIUM)** +### Scope: `blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -765,11 +765,11 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: notes **(PREMIUM)** +### Scope: `notes` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -801,7 +801,7 @@ Example response: ] ``` -### Scope: users +### Scope: `users` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/3/search?scope=users&search=doe" @@ -837,17 +837,17 @@ GET /projects/:id/search | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `scope` | string | yes | The scope to search in | | `search` | string | yes | The search query | -| `ref` | string | no | The name of a repository branch or tag to search on. The project's default branch is used by default. This is only applicable for scopes: commits, blobs, and wiki_blobs. | +| `ref` | string | no | The name of a repository branch or tag to search on. The project's default branch is used by default. This is only applicable for scopes: `commits`, `blobs`, and `wiki_blobs`. | | `state` | string | no | Filter by state. Issues and merge requests are supported; it is ignored for other scopes. | | `confidential` | boolean | no | Filter by confidentiality. Issues scope is supported; it is ignored for other scopes. | | `order_by` | string | no | Allowed values are `created_at` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| | `sort` | string | no | Allowed values are `asc` or `desc` only. If this is not set, the results are either sorted by `created_at` in descending order for basic search, or by the most relevant documents when using advanced search.| -Search the expression in the specified scope. These scopes are supported: issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs, users. +Search the expression in the specified scope. These scopes are supported: `issues`, `merge_requests`, `milestones`, `notes`, `wiki_blobs`, `commits`, `blobs`, `users`. The response depends on the requested scope. -### Scope: issues +### Scope: `issues` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=issues&search=file" @@ -913,7 +913,7 @@ Example response: NOTE: `assignee` column is deprecated, now we show it as a single-sized array `assignees` to conform to the GitLab EE API. -### Scope: merge_requests +### Scope: `merge_requests` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=merge_requests&search=file" @@ -992,7 +992,7 @@ Example response: ] ``` -### Scope: milestones +### Scope: `milestones` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/12/search?scope=milestones&search=release" @@ -1017,11 +1017,11 @@ Example response: ] ``` -### Scope: notes **(PREMIUM)** +### Scope: `notes` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=notes&search=maxime" @@ -1053,11 +1053,11 @@ Example response: ] ``` -### Scope: wiki_blobs **(PREMIUM)** +### Scope: `wiki_blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -1101,11 +1101,11 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` are intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: commits **(PREMIUM)** +### Scope: `commits` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=commits&search=bye" @@ -1136,11 +1136,11 @@ Example response: ] ``` -### Scope: blobs **(PREMIUM)** +### Scope: `blobs` **(PREMIUM)** > Moved to GitLab Premium in 13.9. -This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled. +This scope is available only if [Elasticsearch](../integration/advanced_search/elasticsearch.md) is enabled. Filters are available for this scope: @@ -1183,7 +1183,7 @@ Example response: NOTE: `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` is intended to be only the filename and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/-/issues/34521). -### Scope: users +### Scope: `users` ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/6/search?scope=users&search=doe" diff --git a/doc/api/secure_files.md b/doc/api/secure_files.md index 3532b5a7aeb..b4fe7bb6dd8 100644 --- a/doc/api/secure_files.md +++ b/doc/api/secure_files.md @@ -100,7 +100,7 @@ Supported attributes: | Attribute | Type | Required | Description | |-----------------|----------------|------------------------|-------------| | `project_id` | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `name` | string | **{check-circle}** Yes | The `name` of the file being uploaded. The file name must be unique within the project. | +| `name` | string | **{check-circle}** Yes | The `name` of the file being uploaded. The filename must be unique within the project. | | `file` | file | **{check-circle}** Yes | The `file` being uploaded (5 MB limit). | Example request: diff --git a/doc/api/settings.md b/doc/api/settings.md index 193f18779f5..805797792de 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -11,6 +11,10 @@ These API calls allow you to read and modify GitLab instance as they appear in `/admin/application_settings/general`. You must be an administrator to perform this action. +Application settings are subject to caching and may not immediately take effect. +By default, GitLab caches application settings for 60 seconds. +For information on how to control the application settings cache for an instance, see [Application cache interval](../administration/application_settings_cache.md). + ## Get current application settings List the current [application settings](#list-of-settings-that-can-be-accessed-via-api-calls) @@ -277,10 +281,10 @@ listed in the descriptions of the relevant settings. | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | -| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. | -| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion by default in new groups. Requires both `delayed_group_deletion` to be true and `deletion_adjourned_period` to be greater than 0. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. | +| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), can only be enabled when `delayed_group_deletion` is true. | +| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), disables and locks the group-level setting for delayed protect deletion when set to `false`. | | `delete_inactive_projects` | boolean | no | Enable inactive project deletion feature. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84519) in GitLab 14.10. [Became operational](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0 (with feature flag `inactive_projects_deletion`, disabled by default). | -| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between 0 and 90. On every update, a hook on `deletion_adjourned_period` sets the value of `delayed_group_deletion` to `true` if `deletion_adjourned_period` is greater than 0 and `false` if `deletion_adjourned_period` is 0. +| `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between `1` and `90`. Defaults to `7`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), a hook on `deletion_adjourned_period` sets the period to `1` on every update, and sets both `delayed_project_deletion` and `delayed_group_deletion` to `false` if the period is `0`. | | `diff_max_patch_bytes` | integer | no | Maximum [diff patch size](../user/admin_area/diff_limits.md), in bytes. | | `diff_max_files` | integer | no | Maximum [files in a diff](../user/admin_area/diff_limits.md). | | `diff_max_lines` | integer | no | Maximum [lines in a diff](../user/admin_area/diff_limits.md). | @@ -385,6 +389,10 @@ listed in the descriptions of the relevant settings. | `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. | | `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | | `password_authentication_enabled_for_web` | boolean | no | Enable authentication for the web interface via a GitLab account password. Default is `true`. | +| `password_number_required` **(PREMIUM)** | boolean | no | Indicates whether passwords require at least one number. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85763) in GitLab 15.1. | +| `password_symbol_required` **(PREMIUM)** | boolean | no | Indicates whether passwords require at least one symbol character. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85763) in GitLab 15.1. | +| `password_uppercase_required` **(PREMIUM)** | boolean | no | Indicates whether passwords require at least one uppercase letter. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85763) in GitLab 15.1. | +| `password_lowercase_required` **(PREMIUM)** | boolean | no | Indicates whether passwords require at least one lowercase letter. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85763) in GitLab 15.1. | | `performance_bar_allowed_group_id` | string | no | (Deprecated: Use `performance_bar_allowed_group_path` instead) Path of the group that is allowed to toggle the performance bar. | | `performance_bar_allowed_group_path` | string | no | Path of the group that is allowed to toggle the performance bar. | | `performance_bar_enabled` | boolean | no | (Deprecated: Pass `performance_bar_allowed_group_path: nil` instead) Allow enabling the performance bar. | diff --git a/doc/api/sidekiq_metrics.md b/doc/api/sidekiq_metrics.md index 2be2e71e551..dedd28d6cf6 100644 --- a/doc/api/sidekiq_metrics.md +++ b/doc/api/sidekiq_metrics.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/api/snippet_repository_storage_moves.md b/doc/api/snippet_repository_storage_moves.md index 81473fdff91..a73542c8505 100644 --- a/doc/api/snippet_repository_storage_moves.md +++ b/doc/api/snippet_repository_storage_moves.md @@ -10,7 +10,7 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49228) in GitLab 13.8. Snippet repositories can be moved between storages. This can be useful when -[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrating-to-gitaly-cluster), for +[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), for example. As snippet repository storage moves are processed, they transition through different states. Values diff --git a/doc/api/suggestions.md b/doc/api/suggestions.md index 0a3e7d54f88..c174c0f5264 100644 --- a/doc/api/suggestions.md +++ b/doc/api/suggestions.md @@ -36,7 +36,7 @@ Example response: "id": 36, "from_line": 10, "to_line": 10, - "appliable": false, + "applicable": false, "applied": true, "from_content": " \"--talk-name=org.freedesktop.\",\n", "to_content": " \"--talk-name=org.free.\",\n \"--talk-name=org.desktop.\",\n" diff --git a/doc/api/templates/gitlab_ci_ymls.md b/doc/api/templates/gitlab_ci_ymls.md index 8a55ee84402..ab97823fe07 100644 --- a/doc/api/templates/gitlab_ci_ymls.md +++ b/doc/api/templates/gitlab_ci_ymls.md @@ -5,9 +5,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab CI YMLs API **(FREE)** +# GitLab CI YAMLs API **(FREE)** -In GitLab, there is an API endpoint available to work with GitLab CI/CD YMLs. For more +In GitLab, there is an API endpoint available to work with GitLab CI/CD YAMLs. For more information on CI/CD pipeline configuration in GitLab, see the [configuration reference documentation](../../ci/yaml/index.md). @@ -135,7 +135,7 @@ Example response: ```json { "name": "Ruby", - "content": "# This file is a template, and might need editing before it works on your project.\n# To contribute improvements to CI/CD templates, please follow the Development guide at:\n# https://docs.gitlab.com/ee/development/cicd/templates.html\n# This specific template is located at:\n# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml\n\n# Official language image. Look for the different tagged releases at:\n# https://hub.docker.com/r/library/ruby/tags/\nimage: ruby:latest\n\n# Pick zero or more services to be used on all builds.\n# Only needed when using a docker container to run your tests in.\n# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service\nservices:\n - mysql:latest\n - redis:latest\n - postgres:latest\n\nvariables:\n POSTGRES_DB: database_name\n\n# Cache gems in between builds\ncache:\n paths:\n - vendor/ruby\n\n# This is a basic example for a gem or script which doesn't use\n# services such as redis or postgres\nbefore_script:\n - ruby -v # Print out ruby version for debugging\n # Uncomment next line if your rails app needs a JS runtime:\n # - apt-get update -q && apt-get install nodejs -yqq\n - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby\n - bundle install -j $(nproc)\n\n# Optional - Delete if not using `rubocop`\nrubocop:\n script:\n - rubocop\n\nrspec:\n script:\n - rspec spec\n\nrails:\n variables:\n DATABASE_URL: \"postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB\"\n script:\n - rails db:migrate\n - rails db:seed\n - rails test\n\n# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk\n# are supported too: https://github.com/travis-ci/dpl\ndeploy:\n stage: deploy\n environment: production\n script:\n - gem install dpl\n - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY\n" + "content": "# This file is a template, and might need editing before it works on your project.\n# To contribute improvements to CI/CD templates, please follow the Development guide at:\n# https://docs.gitlab.com/ee/development/cicd/templates.html\n# This specific template is located at:\n# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml\n\n# Official language image. Look for the different tagged releases at:\n# https://hub.docker.com/r/library/ruby/tags/\nimage: ruby:latest\n\n# Pick zero or more services to be used on all builds.\n# Only needed when using a docker container to run your tests in.\n# Check out: https://docs.gitlab.com/ee/ci/services/index.html\nservices:\n - mysql:latest\n - redis:latest\n - postgres:latest\n\nvariables:\n POSTGRES_DB: database_name\n\n# Cache gems in between builds\ncache:\n paths:\n - vendor/ruby\n\n# This is a basic example for a gem or script which doesn't use\n# services such as redis or postgres\nbefore_script:\n - ruby -v # Print out ruby version for debugging\n # Uncomment next line if your rails app needs a JS runtime:\n # - apt-get update -q \u0026\u0026 apt-get install nodejs -yqq\n - bundle config set --local deployment true # Install dependencies into ./vendor/ruby\n - bundle install -j $(nproc)\n\n# Optional - Delete if not using `rubocop`\nrubocop:\n script:\n - rubocop\n\nrspec:\n script:\n - rspec spec\n\nrails:\n variables:\n DATABASE_URL: \"postgresql://postgres:postgres@postgres:5432/$POSTGRES_DB\"\n script:\n - rails db:migrate\n - rails db:seed\n - rails test\n\n# This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk\n# are supported too: https://github.com/travis-ci/dpl\ndeploy:\n stage: deploy\n environment: production\n script:\n - gem install dpl\n - dpl --provider=heroku --app=$HEROKU_APP_NAME --api-key=$HEROKU_PRODUCTION_KEY\n" } ``` diff --git a/doc/api/usage_data.md b/doc/api/usage_data.md index c7064ebf65b..be816a0f864 100644 --- a/doc/api/usage_data.md +++ b/doc/api/usage_data.md @@ -50,10 +50,10 @@ Example response: ## Export Service Ping SQL queries -This action is available only for the GitLab instance [Administrator](../user/permissions.md) users. +This action is behind the `usage_data_queries_api` feature flag and is available only for the GitLab instance [Administrator](../user/permissions.md) users. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57016) in GitLab 13.11. -> - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - [Deployed behind a feature flag](../user/feature_flags.md) named `usage_data_queries_api`, disabled by default. Return all of the raw SQL queries used to compute Service Ping. @@ -113,8 +113,10 @@ Example response: ## UsageDataNonSqlMetrics API +This action is behind the `usage_data_non_sql_metrics` feature flag and is available only for the GitLab instance [Administrator](../user/permissions.md) users. + > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57050) in GitLab 13.11. -> - [Deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - [Deployed behind a feature flag](../user/feature_flags.md), named `usage_data_non_sql_metrics`, disabled by default. Return all non-SQL metrics data used in the Service ping. diff --git a/doc/api/wikis.md b/doc/api/wikis.md index 0122872becf..ebdaa700aa7 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -70,7 +70,7 @@ GET /projects/:id/wikis/:slug | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) | | `slug` | string | yes | URL encoded slug (a unique string) of the wiki page, such as `dir%2Fpage_name` | | `render_html` | boolean | no | Return the rendered HTML of the wiki page | -| `version` | string | no | Wiki page version sha | +| `version` | string | no | Wiki page version SHA | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/wikis/home" diff --git a/doc/architecture/blueprints/ci_data_decay/decomposition_partitioning_comparison.png b/doc/architecture/blueprints/ci_data_decay/decomposition_partitioning_comparison.png Binary files differnew file mode 100644 index 00000000000..47a53ad9be5 --- /dev/null +++ b/doc/architecture/blueprints/ci_data_decay/decomposition_partitioning_comparison.png diff --git a/doc/architecture/blueprints/ci_data_decay/index.md b/doc/architecture/blueprints/ci_data_decay/index.md index cbd9f5dea7a..8808a526df0 100644 --- a/doc/architecture/blueprints/ci_data_decay/index.md +++ b/doc/architecture/blueprints/ci_data_decay/index.md @@ -135,6 +135,9 @@ builds archival, to make it consistent and reliable. Epic: [Partition CI/CD pipelines database tables](https://gitlab.com/groups/gitlab-org/-/epics/5417). +For more technical details about this topic see +[pipeline data partitioning design](pipeline_partitioning.md). + ### Partition CI/CD builds queuing database tables While working on the [CI/CD Scale](../ci_scale/index.md) blueprint, we have @@ -159,6 +162,9 @@ business rule is present in the product since the inception of GitLab CI. Epic: [Partition CI/CD builds queuing database tables](https://gitlab.com/groups/gitlab-org/-/epics/7438). +For more technical details about this topic see +[pipeline data partitioning design](pipeline_partitioning.md). + ## Principles All the three tracks we will use to implement CI/CD time decay pattern are @@ -224,6 +230,22 @@ All three tracks can be worked on in parallel: In progress. +## Timeline + +- 2021-01-21: Parent [CI Scaling](../ci_scale/) blueprint [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52203) created. +- 2021-04-26: CI Scaling blueprint approved and merged. +- 2021-09-10: CI/CD data time decay blueprint discussions started. +- 2022-01-07: CI/CD data time decay blueprint [merged](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70052). +- 2022-02-01: Blueprint [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79110) with new content and links to epics. +- 2022-02-08: Pipeline partitioning PoC [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186) started. +- 2022-02-23: Pipeline partitioning PoC [successful](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186#note_852704724) +- 2022-03-07: A way to attach an existing table as a partition [found and proven](https://gitlab.com/gitlab-org/gitlab/-/issues/353380#note_865237214). +- 2022-03-23: Pipeline partitioning design [Google Doc](https://docs.google.com/document/d/1ARdoTZDy4qLGf6Z1GIHh83-stG_ZLpqsibjKr_OXMgc) started. +- 2022-03-29: Pipeline partitioning PoC [concluded](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186#note_892674358). +- 2022-04-15: Partitioned pipeline data associations PoC [shipped](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84071). +- 2022-04-30: Additional [benchmarking started](https://gitlab.com/gitlab-org/gitlab/-/issues/361019) to evaluate impact. +- 2022-06-31: [Pipeline partitioning design](pipeline_partitioning.md) document [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87683) merged. + ## Who Proposal: diff --git a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md new file mode 100644 index 00000000000..60b20c50696 --- /dev/null +++ b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md @@ -0,0 +1,478 @@ +--- +stage: none +group: unassigned +comments: false +description: 'Pipeline data partitioning design' +--- + +# Pipeline data partitioning design + +_Disclaimer: The following contains information related to upcoming products, +features, and functionality._ + +_It is important to note that the information presented is for informational +purposes only. Please do not rely on this information for purchasing or +planning purposes._ + +_As with all projects, the items mentioned in this document and linked pages +are subject to change or delay. The development, release and timing of any +products, features, or functionality remain at the sole discretion of GitLab +Inc._ + +## What problem are we trying to solve? + +We want to partition the CI/CD dataset, because some of the database tables are +extremely large, which might be challenging in terms of scaling single node +reads, even after we ship the CI/CD database decomposition. + +We want to reduce the risk of database performance degradation by transforming +a few of the largest database tables into smaller ones using PostgreSQL +declarative partitioning. + +See more details about this effort in [the parent blueprint](index.md). + + + +## How are CI/CD data decomposition, partitioning, and time-decay related? + +CI/CD decomposition is an extraction of a CI/CD database cluster out of the +"main" database cluster, to make it possible to have a different primary +database receiving writes. The main benefit is doubling the capacity for writes +and data storage. The new database cluster will not have to serve reads / +writes for non-CI/CD database tables, so this offers some additional capacity +for reads too. + +CI/CD partitioning is dividing large CI/CD database tables into smaller ones. +This will improve reads capacity on every CI/CD database node, because it is +much less expensive to read data from small tables, than from large +multi-terabytes tables. We can add more CI/CD database replicas to better +handle the increase in the number of SQL queries that are reading data, but we +need partitioning to perform a single read more efficiently. Performance in +other aspects will improve too, because PostgreSQL will be more efficient in +maintaining multiple small tables than in maintaining a very large database +table. + +CI/CD time-decay allows us to benefit from the strong time-decay +characteristics of pipeline data. It can be implemented in many different ways, +but using partitioning to implement time-decay might be especially beneficial. +When implementing a time decay we usually mark data as archived, and migrate it +out of a database to a different place when data is no longer relevant or +needed. Our dataset is extremely large (tens of terabytes), so moving such a +high volume of data is challenging. When time-decay is implemented using +partitioning, we can archive the entire partition (or set of partitions) by +simply updating a single record in one of our database tables. It is one of the +least expensive ways to implement time-decay patterns at a database level. + + + +## Why do we need to partition CI/CD data? + +We need to partition CI/CD data because our database tables storing pipelines, +builds, and artifacts are too large. The `ci_builds` database table size is +currently around 2.5 TB with an index of around 1.4 GB. This is too much and +violates our [principle of 100 GB max size](../database_scaling/size-limits.md). +We also want to [build alerting](https://gitlab.com/gitlab-com/gl-infra/tamland/-/issues/5) +to notify us when this number is exceeded. + +We’ve seen numerous S1 and S2 database-related production environment +incidents, over the last couple of months, for example: + +- S1: 2022-03-17 [Increase in writes in `ci_builds` table](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6625) +- S1: 2021-11-22 [Excessive buffer read in replicas for `ci_job_artifacts`](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5952) +- S2: 2022-04-12 [Transactions detected that have been running for more than 10m](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6821) +- S2: 2022-04-06 [Database contention plausibly caused by excessive `ci_builds` reads](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6773) +- S2: 2022-03-18 [Unable to remove a foreign key on `ci_builds`](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6642) + +We have approximately 50 `ci_*` prefixed database tables, and some of them +would benefit from partitioning. + +A simple SQL query to get this data: + +```sql +WITH tables AS (SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'ci_%') + SELECT table_name, + pg_size_pretty(pg_total_relation_size(quote_ident(table_name))) AS total_size, + pg_size_pretty(pg_relation_size(quote_ident(table_name))) AS table_size, + pg_size_pretty(pg_indexes_size(quote_ident(table_name))) AS index_size, + pg_total_relation_size(quote_ident(table_name)) AS total_size_bytes + FROM tables ORDER BY total_size_bytes DESC; +``` + +See data from March 2022: + +| Table name | Total size | Index size | +|-------------------------|------------|------------| +| `ci_builds` | 3.5 TB | 1 TB | +| `ci_builds_metadata` | 1.8 TB | 150 GB | +| `ci_job_artifacts` | 600 GB | 300 GB | +| `ci_pipelines` | 400 GB | 300 GB | +| `ci_stages` | 200 GB | 120 GB | +| `ci_pipeline_variables` | 100 GB | 20 GB | +| (...around 40 more) | | | + +Based on the table above, it is clear that there are tables with a lot of +stored data. + +While we have almost 50 CI/CD-related database tables, we are initially +interested in partitioning only 6 of them. We can start by partitioning the +most interesting tables in an iterative way, but we also should have a strategy +for partitioning the remaining ones if needed. This document is an attempt to +capture this strategy, describe as many details as possible, to share this +knowledge among engineering teams. + +## How do we want to partition CI/CD data? + +We want to partition the CI/CD tables in iterations. It might not be feasible +to partition all of the 6 initial tables at once, so an iterative strategy +might be necessary. We also want to have a strategy for partitioning the +remaining database tables when it becomes necessary. + +It is also important to avoid large data migrations. We store almost 6 +terabytes of data in the biggest CI/CD tables, in many different columns and +indexes. Migrating this amount of data might be challenging and could cause +instability in the production environment. Due to this concern, we’ve developed +a way to attach an existing database table as a partition zero without downtime +and excessive database locking, what has been demonstrated in one of the +[first proofs of concept](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186). +This makes creation of a partitioned schema possible without a downtime (for +example using a routing table `p_ci_pipelines`), by attaching an existing +`ci_pipelines` table as partition zero without exclusive locking. It will be +possible to use the legacy table as usual, but we can create the next partition +when needed and the `p_ci_pipelines` table will be used for routing queries. To +use the routing table we need to find a good partitioning key. + +Our plan is to use logical partition IDs. We want to start with the +`ci_pipelines` table and create a `partition_id` column with a `DEFAULT` value +of `100` or `1000`. Using a `DEFAULT` value avoids the challenge of backfilling +this value for every row. Adding a `CHECK` constraint prior to attaching the +first partition tells PostgreSQL that we’ve already ensured consistency and +there is no need to check it while holding an exclusive table lock when +attaching this table as a partition to the routing table (partitioned schema +definition). We will increment this value every time we create a new partition +for `p_ci_pipelines`, and the partitioning strategy will be `LIST` +partitioning. + +We will also create a `partition_id` column in the other initial 6 database +tables we want to iteratively partition. After a new pipeline is created, it +will get a `partition_id` assigned, and all the related resources, like builds +and artifacts, will share the same value. We want to add the `partition_id` +column into all 6 problematic tables because we can avoid backfilling this data +when we decide it is time to start partitioning them. + +We want to partition CI/CD data iteratively, so we will start with the +pipelines table, and create at least one, but likely two, partitions. The +pipelines table will be partitioned using the `LIST` partitioning strategy. It +is possible that, after some time, `p_ci_pipelines` will store data in two +partitions with IDs of `100` and `101`. Then we will try partitioning +`ci_builds`. Therefore we might want to use `RANGE` partitioning in +`p_ci_builds` with IDs `100` and `101`, because builds for the two logical +partitions used will still be stored in a single table. + +Physical partitioning and logical partitioning will be separated, and a +strategy will be determined when we implement partitioning for the respective +database tables. Using `RANGE` partitioning works similarly to using `LIST` +partitioning in database tables other than `ci_pipelines`, but because we can +guarantee continuity of `partition_id` values, using `RANGE` partitioning might +be a better strategy. + +## Why do we want to use explicit logical partition ids? + +Partitioning CI/CD data using a logical `partition_id` has several benefits. We +could partition by a primary key, but this would introduce much more complexity +and additional cognitive load required to understand how the data is being +structured and stored in partitions. + +CI/CD data is hierarchical data. Stages belong to pipelines, builds belong to +stages, artifacts belong to builds (with rare exceptions). We are designing a +partitioning strategy that reflects this hierarchy, to reduce the complexity +and therefore cognitive load for contributors. With an explicit `partition_id` +associated with a pipeline, we can cascade the partition ID number when trying +to retrieve all resources associated with a pipeline. We know that for a +pipeline `12345` with a `partition_id` of `102`, we are always able to find +associated resources in logical partitions with number `102` in other routing +tables, and PostgreSQL will know in which partitions these records are being +stored in for every table. + +Another interesting benefit for using a single and incremental latest +`partition_id` number, associated with pipelines, is that in theory we can +cache it in Redis or in memory to avoid excessive reads from the database to +find this number, though we might not need to do this. + +The single and uniform `partition_id` value for pipeline data gives us more +choices later on than primary-keys-based partitioning. + +## Splitting large partitions into smaller ones + +We want to start with the initial `pipeline_id` number `100` (or higher, like +`1000`, depending on our calculations and estimations). We do not want to start +from 1, because existing tables are also large already, and we might want to +split them into smaller partitions. If we start with `100`, we will be able to +create partitions for `partition_id` of `1`, `20`, `45`, and move existing +records there by updating `partition_id` from `100` to a smaller number. + +PostgreSQL will move these records into their respective partitions in a +consistent way, provided that we do it in a transaction for all pipeline +resources at the same time. If we ever decide to split large partitions into +smaller ones (it's not yet clear if we will need to do this), we might be able +to just use background migrations to update partition IDs, and PostgreSQL is +smart enough to move rows between partitions on its own. + +## Storing partitions metadata in the database + +In order to build an efficient mechanism that will be responsible for creating +new partitions, and to implement time decay we want to introduce a partitioning +metadata table, called `ci_partitions`. In that table we would store metadata +about all the logical partitions, with many pipelines per partition. We may +need to store a range of pipeline ids per logical partition. Using it we will +be able to find the `partition_id` number for a given pipeline ID and we will +also find information about which logical partitions are “active” or +“archived”, which will help us to implement a time-decay pattern using database +declarative partitioning. + +`ci_partitions` table will store information about a partition identifier, +pipeline ids range it is valid for and whether the partitions have been +archived or not. Additional columns with timestamps may be helpful too. + +## Implementing a time-decay pattern using partitioning + +We can use `ci_partitions` to implement a time-decay pattern using declarative +partitioning. By telling PostgreSQL which logical partitions are archived we +can stop reading from these partitions using a SQL query like the one below. + +```sql +SELECT * FROM ci_builds WHERE partition_id IN ( + SELECT id FROM ci_partitions WHERE active = true +); +``` + +This query will make it possible to limit the number of partitions we will read +from, and therefore will cut access to "archived" pipeline data, using our data +retention policy for CI/CD data. Ideally we do not want to read from more than +two partitions at once, so we need to align the automatic partitioning +mechanisms with the time-decay policy. We will still need to implement new +access patterns for the archived data, presumably through the API, but the cost +of storing archived data in PostgreSQL will be reduced significantly this way. + +There are some technical details here that are out of the scope of this +description, but by using this strategy we can "archive" data, and make it much +less expensive to reside in our PostgreSQL cluster by simply toggling a boolean +column value. + +## Accessing partitioned data + +It will be possible to access partitioned data whether it has been archived or +not, in most places in GitLab. On a merge request page, we will always show +pipeline details even if the merge request was created years ago. We can do +that because `ci_partitions` will be a lookup table associating a pipeline ID +with its `partition_id`, and we will be able to find the partition that the +pipeline data is stored in. + +We will need to constrain access to searching through pipelines, builds, +artifacts etc. Search can not be done through all partitions, as it would not +be efficient enough, hence we will need to find a better way of searching +through archived pipelines data. It will be necessary to have different access +patterns to access archived data in the UI and API. + +There are a few challenges in enforcing usage of the `partition_id` +partitioning key in PostgreSQL. To make it easier to update our application to +support this, we have designed a new queries analyzer in our +[proof of concept merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186). +It helps to find queries that are not using the partitioning key. + +In a [separate proof of concept merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84071) +and [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/357090) we +demonstrated that using the uniform `partition_id` makes it possible to extend +Rails associations with an additional scope modifier so we can provide the +partitioning key in the SQL query. + +Using instance dependent associations, we can easily append a partitioning key +to SQL queries that are supposed to retrieve associated pipeline resources, for +example: + +```ruby +has_many :builds, -> (pipeline) { where(partition_id: pipeline.partition_id) } +``` + +The problem with this approach is that it makes preloading much more difficult +as instance dependent associations can not be used with preloads: + +```plaintext +ArgumentError: The association scope 'builds' is instance dependent (the +scope block takes an argument). Preloading instance dependent scopes is not +supported. +``` + +We also need to build a proof of concept for removing data on the PostgreSQL +side (using foreign keys with `ON DELETE CASCADE`) and removing data through +Rails associations, as this might be an important area of uncertainty. + +We need to [better understand](https://gitlab.com/gitlab-org/gitlab/-/issues/360148) +how unique constraints we are currently using will perform when using the +partitioned schema. + +We have also designed a query analyzer that makes it possible to detect direct +usage of zero partitions, legacy tables that have been attached as first +partitions to routing tables, to ensure that all queries are targeting +partitioned schema or partitioned routing tables, like `p_ci_pipelines`. + +## Why not partition using the project or namespace ID? + +We do not want to partition using `project_id` or `namespace_id` because +sharding and podding is a different problem to solve, on a different layer of +the application. It doesn't solve the original problem statement of performance +growing worse over time as we build up infrequently read data. We may want to +introduce pods in the future, and that might become the primary mechanism of +separating data based on the group or project the data is associated with. + +In theory we could use either `project_id` or `namespace_id` as a second +partitioning dimension, but this would add more complexity to a problem that is +already very complex. + +## Partitioning builds queuing tables + +We also want to partition our builds queuing tables. We currently have two: +`ci_pending_builds` and `ci_running_builds`. These tables are different from +other CI/CD data tables, as there are business rules in our product that make +all data stored in them invalid after 24 hours. + +As a result, we will need to use a different strategy to partition those +database tables, by removing partitions entirely after these are older than 24 +hours, and always reading from two partitions through a routing table. The +strategy to partition these tables is well understood, but requires a solid +Ruby-based automation to manage the creation and deletion of these partitions. +To achieve that we will collaborate with the Database team to adapt +[existing database partitioning tools](../../../development/database/table_partitioning.md) +to support CI/CD data partitioning. + +## Iterating to reduce the risk + +This strategy should reduce the risk of implementing CI/CD partitioning to +acceptable levels. We are also focusing on implementing partitioning for +reading only from two partitions initially to make it possible to detach zero +partitions in case of problems in our production environment. Every iteration +phase, described below has a revert strategy and before shipping database +changes we want to test them in our benchmarking environment. + +The main way of reducing risk in case of this effort is iteration and making +things reversible. Shipping changes, described in this document, in a safe and +reliable way is our priority. + +As we move forward with the implementation we will need to find even more ways +to iterate on the design, support incremental rollouts and have better control +over reverting changes in case of something going wrong. It is sometimes +challenging to ship database schema changes iteratively, and even more +difficult to support incremental rollouts to the production environment. This +can, however, be done, it just sometimes requires additional creativity, that +we will certainly need here. Some examples of how this could look like: + +### Incremental rollout of partitioned schema + +Once we introduce a first partitioned routing table (presumably +`p_ci_pipelines`) and attach its zero partition (`ci_pipelines`), we will need +to start interacting with the new routing table, instead of a concrete +partition zero. Usually we would override the database table the `Ci::Pipeline` +Rails model would use with something like `self.table_name = 'p_ci_pipelines'`. +Unfortunately this approach might not support incremental rollout, because +`self.table_name` will be read upon application boot up, and later we might be +unable revert this change without restarting the application. + +One way of solving this might be introducing `Ci::Partitioned::Pipeline` model, +that will inherit from `Ci::Pipeline`. In that model we would set +`self.table_name` to `p_ci_pipeline` and return its meta class from +`Ci::Pipeline.partitioned` as a scope. This will allow us to use feature flags +to route reads from `ci_pipelines` to `p_ci_pipelines` with a simple revert +strategy. + +### Incremental experimentation with partitioned reads + +Another example would be related to the time when we decide to attach another +partition. The goal of Phase 1 will be have two partitions per partitioned +schema / routing table, meaning that for `p_ci_pipelines` we will have +`ci_pipelines` attached as partition zero, and a new `ci_pipelines_p1` +partition created for new data. All reads from `p_ci_pipelines` will also need +to read data from the `p1` partition and we should also iteratively experiment +with reads targeting more than one partition, to evaluate performance and +overhead of partitioning. + +We can do that by moving _old_ data to `ci_pipelines_m1` (minus 1) partition +iteratively. Perhaps we will create `partition_id = 1` and move some really old +pipelines there. We can then iteratively migrate data into `m1` partition to +measure the impact, performance and increase our confidence before creating a +new partition `p1` for _new_ (still not created) data. + +## Iterations + +We want to focus on Phase 1 iteration first. The goal and the main objective of +this iteration is to partition the biggest 6 CI/CD database tables into 6 +routing tables (partitioned schema) and 12 partitions. This will leave our +Rails SQL queries mostly unchanged, but it will also make it possible to +perform emergency detachment of "zero partitions" if there is a database +performance degradation. This will cut users off their old data, but the +application will remain up and running, which is a better alternative to +application-wide outage. + +1. **Phase 0**: Build CI/CD data partitioning strategy: Done. ✅ +1. **Phase 1**: Partition the 6 biggest CI/CD database tables. + 1. Create partitioned schemas for all 6 database tables. + 1. Design a way to cascade `partition_id` to all partitioned resources. + 1. Implement initial query analyzers validating that we target routing tables. + 1. Attach zero partitions to the partitioned database tables. + 1. Update the application to target routing tables and partitioned tables. + 1. Measure the performance and efficiency of this solution. + + **Revert strategy**: Switch back to using concrete partitions instead of routing tables. + +1. **Phase 2**: Add a partitioning key to add SQL queries targeting partitioned tables. + 1. Implement query analyzer to check if queries targeting partitioned tables + are using proper partitioning keys. + 1. Modify existing queries to make sure that all of them are using a + partitioning key as a filter. + + **Revert strategy**: Use feature flags, query by query. + +1. **Phase 3**: Build new partitioned data access patterns. + 1. Build a new API or extend an existing one to allow access to data stored in + partitions that are supposed to be excluded based on the time-decay data + retention policy. + + **Revert strategy**: Feature flags. + +1. **Phase 4**: Introduce time-decay mechanisms built on top of partitioning. + 1. Build time-decay policy mechanisms. + 1. Enable the time-decay strategy on GitLab.com. +1. **Phase 5**: Introduce mechanisms for creating partitions automatically. + 1. Make it possible to create partitions in an automatic way. + 1. Deliver the new architecture to self-managed instances. + +## Conclusions + +We want to build a solid strategy for partitioning CI/CD data. We are aware of +the fact that it is difficult to iterate on this design, because a mistake made +in managing the database schema of our multi-terabyte PostgreSQL instance might +not be easily reversible without potential downtime. That is the reason we are +spending a significant amount of time to research and refine our partitioning +strategy. The strategy, described in this document, is subject to iteration as +well. Whenever we find a better way to reduce the risk and improve our plan, we +should update this document as well. + +We’ve managed to find a way to avoid large-scale data migrations, and we are +building an iterative strategy for partitioning CI/CD data. We documented our +strategy here to share knowledge and solicit feedback from other team members. + +## Who + +Authors: + +<!-- vale gitlab.Spelling = NO --> + +| Role | Who | +|--------|----------------| +| Author | Grzegorz Bizon | + +Recommenders: + +| Role | Who | +|------------------------|-----------------| +| Distingiushed Engineer | Kamil Trzciński | + +<!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index be21f824392..1c3aee2f860 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -37,7 +37,7 @@ to sustain future growth. ### We are running out of the capacity to store primary keys The primary key in `ci_builds` table is an integer generated in a sequence. -Historically, Rails used to use [integer](https://www.postgresql.org/docs/9.1/datatype-numeric.html) +Historically, Rails used to use [integer](https://www.postgresql.org/docs/14/datatype-numeric.html) type when creating primary keys for a table. We did use the default when we [created the `ci_builds` table in 2012](https://gitlab.com/gitlab-org/gitlab/-/blob/046b28312704f3131e72dcd2dbdacc5264d4aa62/db/ci/migrate/20121004165038_create_builds.rb). [The behavior of Rails has changed](https://github.com/rails/rails/pull/26266) @@ -55,8 +55,8 @@ that have the same problem. Primary keys problem will be tackled by our Database Team. -**Status**: As of October 2021 the primary keys in CI tables have been migrated -to big integers. +**Status**: In October 2021, the primary keys in CI tables were migrated +to big integers. See the [related Epic](https://gitlab.com/groups/gitlab-org/-/epics/5657) for more details. ### The table is too large diff --git a/doc/architecture/blueprints/database/scalability/patterns/index.md b/doc/architecture/blueprints/database/scalability/patterns/index.md index dadf3634407..4a9bb003763 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/index.md +++ b/doc/architecture/blueprints/database/scalability/patterns/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false diff --git a/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md b/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md index 02b56841507..0780ae3c4d5 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md +++ b/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -114,7 +114,7 @@ consider in this pattern (see [#327483](https://gitlab.com/gitlab-org/gitlab/-/i To reduce the database overhead, we implement a cache for the data and thus significantly reduce the query frequency on the database side. There are different scopes for caching available: -- `RequestStore`: per-request in-memory cache (based on [request_store gem](https://github.com/steveklabnik/request_store)) +- `RequestStore`: per-request in-memory cache (based on [`request_store` gem](https://github.com/steveklabnik/request_store)) - [`ProcessMemoryCache`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/process_memory_cache.rb#L4): per-process in-memory cache (a `ActiveSupport::Cache::MemoryStore`) - [`Gitlab::Redis::Cache`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/redis/cache.rb) and `Rails.cache`: full-blown cache in Redis diff --git a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md index b4614cde9d4..7a64f0cb7c6 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md +++ b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -71,7 +71,7 @@ The second and most important characteristic of time-decay data is that most of able to implicitly or explicitly access the data using a date filter, **restricting our results based on a time-related dimension**. -There can be many such dimensions, but we are only going to focus on the creation date as it is both +There can be many such dimensions, but we focus only on the creation date as it is both the most commonly used, and the one that we can control and optimize against. It: - Is immutable. @@ -107,7 +107,7 @@ perspective, but that definition is volatile and not actionable. Finally, a characteristic that further differentiates time-decay data in sub-categories with slightly different approaches available is **whether we want to keep the old data or not** (for example, retention policy) and/or -**whether old data will be accessible by users through the application**. +**whether old data is accessible by users through the application**. #### (optional) Extended definition of time-decay data @@ -148,7 +148,7 @@ factors: would include too many unnecessary records in each partition, as is the case for `web_hook_logs`. 1. **How large are the partitions created?** The major purpose of partitioning is accessing tables that are as small as possible. If they get too - large by themselves, queries will start underperforming. We may have to re-partition (split) them + large by themselves, queries start underperforming. We may have to re-partition (split) them in even smaller partitions. The perfect partitioning scheme keeps **all queries over a dataset almost always over a single partition**, @@ -194,7 +194,7 @@ The disadvantage of such a solution over large, non-partitioned tables is that w access and delete all the records that are considered as not relevant any more. That is a very expensive operation, due to multi-version concurrency control in PostgreSQL. It also leads to the pruning worker not being able to catch up with new records being created, if that rate exceeds a -threshold, as is the case of [web_hook_logs](https://gitlab.com/gitlab-org/gitlab/-/issues/256088) +threshold, as is the case of [`web_hook_logs`](https://gitlab.com/gitlab-org/gitlab/-/issues/256088) at the time of writing this document. For the aforementioned reasons, our proposal is that @@ -315,7 +315,7 @@ The process required follows: 1. After the non-partitioned table is dropped, we can add a worker to implement the pruning strategy by dropping past partitions. - In this case, the worker will make sure that only 4 partitions are always active (as the + In this case, the worker makes sure that only 4 partitions are always active (as the retention policy is 90 days) and drop any partitions older than four months. We have to keep 4 months of partitions while the current month is still active, as going 90 days back takes you to the fourth oldest partition. @@ -325,7 +325,7 @@ The process required follows: Related epic: [Partitioning: Design and implement partitioning strategy for Audit Events](https://gitlab.com/groups/gitlab-org/-/epics/3206) The `audit_events` table shares a lot of characteristics with the `web_hook_logs` table discussed -in the previous sub-section, so we are going to focus on the points they differ. +in the previous sub-section, so we focus on the points they differ. The consensus was that [partitioning could solve most of the performance issues](https://gitlab.com/groups/gitlab-org/-/epics/3206#note_338157248). diff --git a/doc/architecture/blueprints/database_scaling/size-limits.md b/doc/architecture/blueprints/database_scaling/size-limits.md index a0508488620..375c82f8833 100644 --- a/doc/architecture/blueprints/database_scaling/size-limits.md +++ b/doc/architecture/blueprints/database_scaling/size-limits.md @@ -1,17 +1,18 @@ --- +stage: Data Stores +group: Database comments: false description: 'Database Scalability / Limit table sizes' -group: database --- # Database Scalability: Limit on-disk table size to < 100 GB for GitLab.com -This document is a proposal to work towards reducing and limiting table sizes on GitLab.com. We establish a **measurable target** by limiting table size to a certain threshold. This will be used as an indicator to drive database focus and decision making. With GitLab.com growing, we continuously re-evaluate which tables need to be worked on to prevent or otherwise fix violations. +This document is a proposal to work towards reducing and limiting table sizes on GitLab.com. We establish a **measurable target** by limiting table size to a certain threshold. This is used as an indicator to drive database focus and decision making. With GitLab.com growing, we continuously re-evaluate which tables need to be worked on to prevent or otherwise fix violations. This is not meant to be a hard rule but rather a strong indication that work needs to be done to break a table apart or otherwise reduce its size. This is meant to be read in context with the [Database Sharding blueprint](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64115), -which paints the bigger picture. This proposal here is thought to be part of the "debloating step" below, as we aim to reduce storage requirements and improve data modeling. Partitioning is part of the standard tool-belt: where possible, we can already use partitioning as a solution to cut physical table sizes significantly. Both will help to prepare efforts like decomposition (database usage is already optimized) and sharding (database is already partitioned along an identified data access dimension). +which paints the bigger picture. This proposal here is thought to be part of the "debloating step" below, as we aim to reduce storage requirements and improve data modeling. Partitioning is part of the standard tool-belt: where possible, we can already use partitioning as a solution to cut physical table sizes significantly. Both help to prepare efforts like decomposition (database usage is already optimized) and sharding (database is already partitioned along an identified data access dimension). ```mermaid graph LR @@ -36,7 +37,7 @@ Large tables on GitLab.com are a major problem - for both operations and develop 1. **Table maintenance** becomes much more costly. Vacuum activity has become a significant concern on GitLab.com - with large tables only seeing infrequent (once per day) processing and vacuum runs taking many hours to complete. This has various negative consequences and a very large table has potential to impact seemingly unrelated parts of the database and hence overall application performance suffers. 1. **Data migrations** on large tables are significantly more complex to implement and incur development overhead. They have potential to cause stability problems on GitLab.com and take a long time to execute on large datasets. 1. **Indexes size** is significant. This directly impacts performance as smaller parts of the index are kept in memory and also makes the indexes harder to maintain (think repacking). -1. **Index creation times** go up significantly - in 2021, we see btree creation take up to 6 hours for a single btree index. This impacts our ability to deploy frequently and leads to vacuum-related problems (delayed cleanup). +1. **Index creation times** go up significantly - in 2021, we see B-Tree creation take up to 6 hours for a single B-Tree index. This impacts our ability to deploy frequently and leads to vacuum-related problems (delayed cleanup). 1. We tend to add **many indexes** to mitigate, but this eventually causes significant overhead, can confuse the query planner and a large number of indexes is a smell of a design problem. ## Examples @@ -124,11 +125,11 @@ In order to maintain and improve operational stability and lessen development bu 1. Indexes are smaller, can be maintained more efficiently and fit better into memory 1. Data migrations are easier to reason about, take less time to implement and execute -This target is *pragmatic*: We understand table sizes depend on feature usage, code changes and other factors - which all change over time. We may not always find solutions where we can tightly limit the size of physical tables once and for all. That is acceptable though and we primarily aim to keep the situation on GitLab.com under control. We adapt our efforts to the situation present on GitLab.com and will re-evaluate frequently. +This target is *pragmatic*: We understand table sizes depend on feature usage, code changes and other factors - which all change over time. We may not always find solutions where we can tightly limit the size of physical tables once and for all. That is acceptable though and we primarily aim to keep the situation on GitLab.com under control. We adapt our efforts to the situation present on GitLab.com and re-evaluate frequently. -While there are changes we can make that lead to a constant maximum physical table size over time, this doesn't need to be the case necessarily. Consider for example hash partitioning, which breaks a table down into a static number of partitions. With data growth over time, individual partitions will also grow in size and may eventually reach the threshold size again. We strive to get constant table sizes, but it is acceptable to ship easier solutions that don't have this characteristic but improve the situation for a considerable amount of time. +While there are changes we can make that lead to a constant maximum physical table size over time, this doesn't need to be the case necessarily. Consider for example hash partitioning, which breaks a table down into a static number of partitions. With data growth over time, individual partitions also grow in size and may eventually reach the threshold size again. We strive to get constant table sizes, but it is acceptable to ship easier solutions that don't have this characteristic but improve the situation for a considerable amount of time. -As such, the target size of a physical table after refactoring depends on the situation and there is no hard rule for it. We suggest to consider historic data growth and forecast when physical tables will reach the threshold of 100 GB again. This allows us to understand how long a particular solution is expected to last until the model has to be revisited. +As such, the target size of a physical table after refactoring depends on the situation and there is no hard rule for it. We suggest to consider historic data growth and forecast when physical tables reach the threshold of 100 GB again. This allows us to understand how long a particular solution is expected to last until the model has to be revisited. ## Solutions @@ -153,10 +154,10 @@ For solutions like normalization, this is a trade-off: Denormalized models can s A few examples can be found below, many more are organized under the epic [Database efficiency](https://gitlab.com/groups/gitlab-org/-/epics/5585). 1. [Reduce number of indexes on `ci_builds`](https://gitlab.com/groups/gitlab-org/-/epics/6203) -1. [Normalize and de-duplicate committer and author details in merge_request_diff_commits](https://gitlab.com/gitlab-org/gitlab/-/issues/331823) +1. [Normalize and de-duplicate committer and author details in `merge_request_diff_commits`](https://gitlab.com/gitlab-org/gitlab/-/issues/331823) 1. [Retention strategy for `ci_build_trace_sections`](https://gitlab.com/gitlab-org/gitlab/-/issues/32565#note_603138100) 1. [Implement worker that hard-deletes old CI jobs metadata](https://gitlab.com/gitlab-org/gitlab/-/issues/215646) -1. [merge_request_diff_files violates < 100 GB target](https://gitlab.com/groups/gitlab-org/-/epics/6215) (epic) +1. [`merge_request_diff_files` violates < 100 GB target](https://gitlab.com/groups/gitlab-org/-/epics/6215) (epic) ## Goal diff --git a/doc/architecture/blueprints/database_testing/index.md b/doc/architecture/blueprints/database_testing/index.md index 22857abf176..30ebd06c81f 100644 --- a/doc/architecture/blueprints/database_testing/index.md +++ b/doc/architecture/blueprints/database_testing/index.md @@ -20,7 +20,7 @@ Developers are expected to test database migrations prior to deploying to any en The [code review phase](../../../development/database_review.md) involves Database Reviewers and Maintainers to manually check the migrations committed. This often involves knowing and spotting problematic patterns and their particular behavior on GitLab.com from experience. There is no large-scale environment available that allows us to test database migrations before they are being merged. -Testing in CI is done on a very small database. We mainly check forward/backward migration consistency, evaluate Rubocop rules to detect well-known problematic behaviors (static code checking) and have a few other, rather technical checks in place (adding the right files etc). That is, we typically find code or other rather simple errors, but cannot surface any data related errors - which are also typically not covered by unit tests either. +Testing in CI is done on a very small database. We mainly check forward/backward migration consistency, evaluate RuboCop rules to detect well-known problematic behaviors (static code checking) and have a few other, rather technical checks in place (adding the right files etc). That is, we typically find code or other rather simple errors, but cannot surface any data related errors - which are also typically not covered by unit tests either. Once merged, migrations are being deployed to the staging environment. Its database size is less than 5% of the production database size as of January 2021 and its recent data distribution does not resemble the production site. Oftentimes, we see migrations succeed in staging but then fail in production due to query timeouts or other unexpected problems. Even if we caught problems in staging, this is still expensive to reconcile and ideally we want to catch those problems as early as possible in the development cycle. @@ -127,7 +127,7 @@ An alternative approach we have discussed and abandoned is to "scrub" and anonym <!-- vale gitlab.Spelling = NO --> -This effort is owned and driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/database/) with support from the [GitLab.com Reliability Datastores](https://about.gitlab.com/handbook/engineering/infrastructure/team/reliability/datastores/) team. +This effort is owned and driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/database/) with support from the [GitLab.com Reliability Datastores](https://about.gitlab.com/handbook/engineering/infrastructure/team/reliability/) team. | Role | Who |------------------------------|-------------------------| @@ -142,7 +142,6 @@ DRIs: | Role | Who |------------------------------|------------------------| | Product | Fabian Zimmer | -| Leadership | Craig Gomes | | Engineering | Andreas Brandl | <!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md index 174fe191cc7..c4bd8433ab3 100644 --- a/doc/architecture/blueprints/runner_scaling/index.md +++ b/doc/architecture/blueprints/runner_scaling/index.md @@ -36,7 +36,7 @@ rapid growth of CI/CD adoption on GitLab.com. We can not, however, continue using Docker Machine. Work on that project [was paused in July 2018](https://github.com/docker/machine/issues/4537) and there was no development made since that time (except for some highly important -security fixes). In 2018, after Docker Machine entered the “maintenance mode”, +security fixes). In 2018, after Docker Machine entered the "maintenance mode", we decided to create [our own fork](https://gitlab.com/gitlab-org/ci-cd/docker-machine) to be able to keep using this and ship fixes and updates needed for our use case. [On September 26th, 2021 the project got archived](https://github.com/docker/docker.github.io/commit/2dc8b49dcbe85686cc7230e17aff8e9944cb47a5) @@ -48,7 +48,7 @@ new mechanism for GitLab Runner auto-scaling. It not only needs to support auto-scaling, but it also needs to do that in the way to enable us to build on top of it to improve efficiency, reliability and availability. -We call this new mechanism the “next GitLab Runner Scaling architecture”. +We call this new mechanism the "next GitLab Runner Scaling architecture". _Disclaimer The following contain information related to upcoming products, features, and functionality._ @@ -82,11 +82,11 @@ about how people are using Docker Machine right now, and it seems that GitLab CI is one of the most frequent reasons for people to keep using Docker Machine. There is also an opportunity in being able to optionally run multiple jobs in a -single, larger virtual machine. We can’t do that today, but we know that this +single, larger virtual machine. We can't do that today, but we know that this can potentially significantly improve efficiency. We might want to build a new architecture that makes it easier and allows us to test how efficient it is with PoCs. Running multiple jobs on a single machine can also make it possible -to reuse what we call a “sticky context” - a space for build artifacts / user +to reuse what we call a "sticky context" - a space for build artifacts / user data that can be shared between job runs. ### 💡 Design a simple abstraction that users will be able to build on top of @@ -165,7 +165,7 @@ sequence diagram.  On the diagrams above we see that currently a GitLab Runner Manager runs on a -machine that has access to a cloud provider’s API. It is using Docker Machine +machine that has access to a cloud provider's API. It is using Docker Machine to provision new Virtual Machines with Docker Engine installed and it configures the Docker daemon there to allow external authenticated requests. It stores credentials to such ephemeral Docker environments on disk. Once a @@ -186,8 +186,8 @@ through os/exec system calls. Thanks to the custom executor abstraction there is no more need to implement new executors internally in Runner. Users who have specific needs can implement -their own drivers and don’t need to wait for us to make their work part of the -“official” GitLab Runner. As each driver is a separate project, it also makes +their own drivers and don't need to wait for us to make their work part of the +"official" GitLab Runner. As each driver is a separate project, it also makes it easier to create communities around them, where interested people can collaborate together on improvements and bug fixes. @@ -197,7 +197,7 @@ provide a context and an environment in which a build will be executed by one of the Custom Executors. There are multiple solutions to implementing a custom provider abstraction. We -can use raw Go plugins, Hashcorp’s Go Plugin, HTTP interface or gRPC based +can use raw Go plugins, Hashcorp's Go Plugin, HTTP interface or gRPC based facade service. There are many solutions, and we want to choose the most optimal one. In order to do that, we will describe the solutions in a separate document, define requirements and score the solution accordingly. This will diff --git a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md index 0aa7f157602..40b29ebb6ea 100644 --- a/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md +++ b/doc/ci/ci_cd_for_external_repos/bitbucket_integration.md @@ -17,9 +17,7 @@ To use GitLab CI/CD with a Bitbucket Cloud repository: 1. In GitLab, create a project: 1. On the top menu, select **Projects > Create new project**. 1. Select **Run CI/CD for external repository**. - 1. Select **Repo by URL**. - -  + 1. Select **Repository by URL**. GitLab imports the repository and enables [Pull Mirroring](../../user/project/repository/mirror/pull.md). diff --git a/doc/ci/ci_cd_for_external_repos/github_integration.md b/doc/ci/ci_cd_for_external_repos/github_integration.md index a906f213626..a928d315c6b 100644 --- a/doc/ci/ci_cd_for_external_repos/github_integration.md +++ b/doc/ci/ci_cd_for_external_repos/github_integration.md @@ -62,7 +62,7 @@ To manually enable GitLab CI/CD for your repository: `repo` so that GitLab can access your project and update commit statuses. 1. In GitLab, create a project: 1. On the top menu, select **Projects > Create new project**. - 1. Select **Run CI/CD for external repository** and **Repo by URL**. + 1. Select **Run CI/CD for external repository** and **Repository by URL**. 1. In the **Git repository URL** field, enter the HTTPS URL for your GitHub repository. If your project is private, use the personal access token you just created for authentication. 1. Fill in all the other fields and select **Create project**. diff --git a/doc/ci/ci_cd_for_external_repos/img/bitbucket_app_password.png b/doc/ci/ci_cd_for_external_repos/img/bitbucket_app_password.png Binary files differindex ac5be3c3058..163148d53fd 100644 --- a/doc/ci/ci_cd_for_external_repos/img/bitbucket_app_password.png +++ b/doc/ci/ci_cd_for_external_repos/img/bitbucket_app_password.png diff --git a/doc/ci/ci_cd_for_external_repos/img/bitbucket_webhook.png b/doc/ci/ci_cd_for_external_repos/img/bitbucket_webhook.png Binary files differindex 0a3476d9035..b2a5c1c1eb8 100644 --- a/doc/ci/ci_cd_for_external_repos/img/bitbucket_webhook.png +++ b/doc/ci/ci_cd_for_external_repos/img/bitbucket_webhook.png diff --git a/doc/ci/ci_cd_for_external_repos/img/external_repository.png b/doc/ci/ci_cd_for_external_repos/img/external_repository.png Binary files differdeleted file mode 100644 index b850d91f56b..00000000000 --- a/doc/ci/ci_cd_for_external_repos/img/external_repository.png +++ /dev/null diff --git a/doc/ci/ci_cd_for_external_repos/index.md b/doc/ci/ci_cd_for_external_repos/index.md index 4e0cd7609cb..e3a8141ed88 100644 --- a/doc/ci/ci_cd_for_external_repos/index.md +++ b/doc/ci/ci_cd_for_external_repos/index.md @@ -18,21 +18,17 @@ external repository to get the benefits of GitLab CI/CD. Connecting an external repository sets up [repository mirroring](../../user/project/repository/mirror/index.md) and creates a lightweight project with issues, merge requests, wiki, and snippets disabled. These features -[can be re-enabled later](../../user/project/settings/index.md#sharing-and-permissions). +[can be re-enabled later](../../user/project/settings/index.md#configure-project-visibility-features-and-permissions). ## Connect to an external repository To connect to an external repository: -<!-- vale gitlab.Spelling = NO --> - 1. On the top bar, select **Menu > Projects > Create new project**. 1. Select **Run CI/CD for external repository**. -1. Select **GitHub** or **Repo by URL**. +1. Select **GitHub** or **Repository by URL**. 1. Complete the fields. -<!-- vale gitlab.Spelling = YES --> - ## Pipelines for external pull requests > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/65139) in GitLab 12.3. diff --git a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md new file mode 100644 index 00000000000..9af5218e058 --- /dev/null +++ b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md @@ -0,0 +1,253 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Deploy to Amazon Elastic Container Service **(FREE)** + +This step-by-step guide helps you deploy a project hosted on GitLab.com to +the Amazon [Elastic Container Service (ECS)](https://aws.amazon.com/ecs/). + +In this guide, you begin by creating an ECS cluster manually using the AWS console. You create and +deploy a simple application that you create from a GitLab template. + +These instructions work for both SaaS and self-managed GitLab instances. +Ensure your own [runners are configured](../../runners/index.md). + +## Prerequisites + +- An [AWS account](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). + Sign in with an existing AWS account or create a new one. +- In this guide, you create an infrastructure in [`us-east-2` region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html). + You can use any region, but do not change it after you begin. + +## Create an infrastructure and initial deployment on AWS + +For deploying an application from GitLab, you must first create an infrastructure and initial +deployment on AWS. +This includes an [ECS cluster](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html) +and related components, such as +[ECS task definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html), +[ECS services](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html), +and containerized application image. + +For the first step here, you create a demo application from a project template. + +### Create a new project from a template + +Use a GitLab project template to get started. As the name suggests, these projects provide a +bare-bones application built on some well-known frameworks. + +1. In GitLab, select the plus icon (**{plus-square}**) at the top of the navigation bar, and select + **New project**. + +1. Select **Create from template**, where you can choose from a Ruby on Rails, Spring, or + NodeJS Express project. For this guide, use the Ruby on Rails template. + +  + +1. Give your project a name. In this example, it's named `ecs-demo`. Make it public so that you can + take advantage of the features available in the + [GitLab Ultimate plan](https://about.gitlab.com/pricing/). + +1. Select **Create project**. + +Now that you created a demo project, you must containerize the application and push it to the +container registry. + +### Push a containerized application image to GitLab Container Registry + +[ECS](https://aws.amazon.com/ecs/) is a container orchestration service, meaning that you must +provide a containerized application image during the infrastructure build. To do so, you can use +GitLab [Auto Build](../../../topics/autodevops/stages.md#auto-build) +and [Container Registry](../../../user/packages/container_registry/index.md). + +1. Go to **ecs-demo** project on GitLab. +1. Select **Setup up CI/CD**. It brings you to a `.gitlab-ci.yml` + creation form. +1. Copy and paste the following content into the empty `.gitlab-ci.yml`. This defines + a pipeline for continuous deployment to ECS. + + ```yaml + include: + - template: AWS/Deploy-ECS.gitlab-ci.yml + ``` + +1. Select **Commit Changes**. It automatically triggers a new pipeline. In this pipeline, the `build` + job containerizes the application and pushes the image to [GitLab Container Registry](../../../user/packages/container_registry/index.md). + +  + +1. Visit **Packages & Registries > Container Registry**. Make sure the application image has been + pushed. + +  + +Now you have a containerized application image that can be pulled from AWS. Next, you define the +spec of how this application image is used in AWS. + +Note that the `production_ecs` job fails because ECS Cluster is not connected yet. You'll fix this +later. + +### Create an ECS task definition + +[ECS Task definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html) +is a specification about how the application image is started by an [ECS service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html). + +1. Go to **ECS > Task Definitions** on [AWS console](https://aws.amazon.com/). +1. Select **Create new Task Definition**. + +  + +1. Choose **EC2** as the launch type. Select **Next Step**. +1. Set `ecs_demo` to **Task Definition Name**. +1. Set `512` to **Task Size > Task memory** and **Task CPU**. +1. Select **Container Definitions > Add container**. This opens a container registration form. +1. Set `web` to **Container name**. +1. Set `registry.gitlab.com/<your-namespace>/ecs-demo/master:latest` to **Image**. + Alternatively, you can copy and paste the image path from the [GitLab Container Registry page](#push-a-containerized-application-image-to-gitlab-container-registry). + +  + +1. Add a port mapping. Set `80` to **Host Port** and `5000` to **Container port**. + +  + +1. Select **Create**. + +Now you have the initial task definition. Next, you create an actual infrastructure to run the +application image. + +### Create an ECS cluster + +An [ECS cluster](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html) +is a virtual group of [ECS services](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html). +It's also associated with EC2 or Fargate as the computation resource. + +1. Go to **ECS > Clusters** on [AWS console](https://aws.amazon.com/). +1. Select **Create Cluster**. +1. Select **EC2 Linux + Networking** as the cluster template. Select **Next Step**. +1. Set `ecs-demo` to **Cluster Name**. +1. Choose the default [VPC](https://aws.amazon.com/vpc/?vpc-blogs.sort-by=item.additionalFields.createdDate&vpc-blogs.sort-order=desc) + in **Networking**. If there are no existing VPCs, you can leave it as-is to create a new one. +1. Set all available subnets of the VPC to **Subnets**. +1. Select **Create**. +1. Make sure that the ECS cluster has been successfully created. + +  + +Now you can register an ECS service to the ECS cluster in the next step. + +Note the following: + +- Optionally, you can set a SSH key pair in the creation form. This allows you to SSH to the EC2 + instance for debugging. +- If you don't choose an existing VPC, it creates a new VPC by default. This could cause an error if + it reaches the maximum allowed number of internet gateways on your account. +- The cluster requires an EC2 instance, meaning it costs you [according to the instance-type](https://aws.amazon.com/ec2/pricing/on-demand/). + +### Create an ECS Service + +[ECS service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html) +is a daemon to create an application container based on the [ECS task definition](#create-an-ecs-task-definition). + +1. Go to **ECS > Clusters > ecs-demo > Services** on the [AWS console](https://aws.amazon.com/) +1. Select **Deploy**. This opens a service creation form. +1. Select `EC2` in **Launch Type**. +1. Set `ecs_demo` to **Task definition**. This corresponds to [the task definition you created above](#create-an-ecs-task-definition). +1. Set `ecs_demo` to **Service name**. +1. Set `1` to **Desired tasks**. + +  + +1. Select **Deploy**. +1. Make sure that the created service is active. + +  + +Note that AWS's console UI changes from time to time. If you can't find a relevant component in the +instructions, select the closest one. + +### View the demo application + +Now, the demo application is accessible from the internet. + +1. Go to **EC2 > Instances** on the [AWS console](https://aws.amazon.com/) +1. Search by `ECS Instance` to find the corresponding EC2 instance that [the ECS cluster created](#create-an-ecs-cluster). +1. Select the ID of the EC2 instance. This brings you to the instance detail page. +1. Copy **Public IPv4 address** and paste it in the browser. Now you can see the demo application + running. + +  + +In this guide, HTTPS/SSL is **not** configured. You can access to the application through HTTP only +(for example, `http://<ec2-ipv4-address>`). + +## Setup Continuous Deployment from GitLab + +Now that you have an application running on ECS, you can set up continuous deployment from GitLab. + +### Create a new IAM user as a deployer + +For GitLab to access the ECS cluster, service, and task definition that you created above, You must +create a deployer user on AWS: + +1. Go to **IAM > Users** on [AWS console](https://aws.amazon.com/). +1. Select **Add user**. +1. Set `ecs_demo` to **User name**. +1. Enable **Programmatic access** checkbox. Select **Next: Permissions**. +1. Select `Attach existing policies directly` in **Set permissions**. +1. Select `AmazonECS_FullAccess` from the policy list. Select **Next: Tags** and **Next: Review**. + +  + +1. Select **Create user**. +1. Take note of the **Access key ID** and **Secret access key** of the created user. + +NOTE: +Do not share the secret access key in a public place. You must save it in a secure place. + +### Setup credentials in GitLab to let pipeline jobs access to ECS + +You can register the access information in [GitLab Environment Variables](../../variables/index.md#custom-cicd-variables). +These variables are injected into the pipeline jobs and can access the ECS API. + +1. Go to **ecs-demo** project on GitLab. +1. Go to **Settings > CI/CD > Variables**. +1. Select **Add Variable** and set the following key-value pairs. + + |Key|Value|Note| + |---|---|---| + |`AWS_ACCESS_KEY_ID`|`<Access key ID of the deployer>`| For authenticating `aws` CLI. | + |`AWS_SECRET_ACCESS_KEY`|`<Secret access key of the deployer>`| For authenticating `aws` CLI. | + |`AWS_DEFAULT_REGION`|`us-east-2`| For authenticating `aws` CLI. | + |`CI_AWS_ECS_CLUSTER`|`ecs-demo`| The ECS cluster is accessed by `production_ecs` job. | + |`CI_AWS_ECS_SERVICE`|`ecs_demo`| The ECS service of the cluster is updated by `production_ecs` job. | + |`CI_AWS_ECS_TASK_DEFINITION`|`ecs_demo`| The ECS task definition is updated by `production_ecs` job. | + +### Make a change to the demo application + +Change a file in the project and see if it's reflected in the demo application on ECS: + +1. Go to **ecs-demo** project on GitLab. +1. Open the file at **app > views > welcome > `index.html.erb`**. +1. Select **Edit**. +1. Change the text to `You're on ECS!`. +1. Select **Commit Changes**. This automatically triggers a new pipeline. Wait until it finishes. +1. [Access the running application on the ECS cluster](#view-the-demo-application). You should see + this: + +  + +Congratulations! You successfully set up continuous deployment to ECS. + +NOTE: +ECS deploy jobs wait for the rollout to complete before exiting. To disable this behavior, +set `CI_AWS_ECS_WAIT_FOR_ROLLOUT_COMPLETE_DISABLED` to a non-empty value. + +## Further reading + +- If you're interested in more of the continuous deployments to clouds, see [cloud deployments](../index.md). +- If you want to quickly set up DevSecOps in your project, see [Auto DevOps](../../../topics/autodevops/index.md). +- If you want to quickly set up the production-grade environment, see [the 5 Minute Production App](https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/master/README.md). diff --git a/doc/ci/cloud_deployment/ecs/quick_start_guide.md b/doc/ci/cloud_deployment/ecs/quick_start_guide.md index 5571c51ef27..7522bc80d0d 100644 --- a/doc/ci/cloud_deployment/ecs/quick_start_guide.md +++ b/doc/ci/cloud_deployment/ecs/quick_start_guide.md @@ -1,250 +1,11 @@ --- -stage: Release -group: Release -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'deploy_to_aws_ecs.md' +remove_date: '2022-09-18' --- -# Getting started with Continuous Deployment to AWS Elastic Container Service **(FREE)** +This document was moved to [another location](deploy_to_aws_ecs.md). -This step-by-step guide helps you use [Continuous Deployment to ECS](../index.md#deploy-your-application-to-the-aws-elastic-container-service-ecs) -that deploys a project hosted on GitLab.com to [Elastic Container Service](https://aws.amazon.com/ecs/) -(ECS) on AWS. - -In this guide, you begin by creating an ECS cluster manually using the AWS console. You create and -deploy a simple application that you create from a GitLab template. - -These instructions work for both SaaS and self-managed GitLab instances. -Ensure your own [runners are configured](../../runners/index.md). - -## Prerequisites - -- An [AWS account](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). - Sign in with an existing AWS account or create a new one. -- In this guide, you create an infrastructure in [`us-east-2` region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html). - You can use any region, but do not change it after you begin. - -## Create an infrastructure and initial deployment on AWS - -For deploying an application from GitLab, you must first create an infrastructure and initial -deployment on AWS. -This includes an [ECS cluster](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html) -and related components, such as -[ECS task definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html), -[ECS services](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html), -and containerized application image. - -For the first step here, you create a demo application from a project template. - -### Create a new project from a template - -Use a GitLab project template to get started. As the name suggests, these projects provide a -bare-bones application built on some well-known frameworks. - -1. In GitLab, select the plus icon (**{plus-square}**) at the top of the navigation bar, and select - **New project**. - -1. Select **Create from template**, where you can choose from a Ruby on Rails, Spring, or - NodeJS Express project. For this guide, use the Ruby on Rails template. - -  - -1. Give your project a name. In this example, it's named `ecs-demo`. Make it public so that you can - take advantage of the features available in the - [GitLab Ultimate plan](https://about.gitlab.com/pricing/). - -1. Select **Create project**. - -Now that you created a demo project, you must containerize the application and push it to the -container registry. - -### Push a containerized application image to GitLab Container Registry - -[ECS](https://aws.amazon.com/ecs/) is a container orchestration service, meaning that you must -provide a containerized application image during the infrastructure build. To do so, you can use -GitLab [Auto Build](../../../topics/autodevops/stages.md#auto-build) -and [Container Registry](../../../user/packages/container_registry/index.md). - -1. Go to **ecs-demo** project on GitLab. -1. Select **Setup up CI/CD**. It brings you to a `.gitlab-ci.yml` - creation form. -1. Copy and paste the following content into the empty `.gitlab-ci.yml`. This defines - [a pipeline for continuous deployment to ECS](../index.md#deploy-your-application-to-the-aws-elastic-container-service-ecs). - - ```yaml - include: - - template: AWS/Deploy-ECS.gitlab-ci.yml - ``` - -1. Select **Commit Changes**. It automatically triggers a new pipeline. In this pipeline, the `build` - job containerizes the application and pushes the image to [GitLab Container Registry](../../../user/packages/container_registry/index.md). - -  - -1. Visit **Packages & Registries > Container Registry**. Make sure the application image has been - pushed. - -  - -Now you have a containerized application image that can be pulled from AWS. Next, you define the -spec of how this application image is used in AWS. - -Note that the `production_ecs` job fails because ECS Cluster is not connected yet. You'll fix this -later. - -### Create an ECS task definition - -[ECS Task definitions](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html) -is a specification about how the application image is started by an [ECS service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html). - -1. Go to **ECS > Task Definitions** on [AWS console](https://aws.amazon.com/). -1. Select **Create new Task Definition**. - -  - -1. Choose **EC2** as the launch type. Select **Next Step**. -1. Set `ecs_demo` to **Task Definition Name**. -1. Set `512` to **Task Size > Task memory** and **Task CPU**. -1. Select **Container Definitions > Add container**. This opens a container registration form. -1. Set `web` to **Container name**. -1. Set `registry.gitlab.com/<your-namespace>/ecs-demo/master:latest` to **Image**. - Alternatively, you can copy and paste the image path from the [GitLab Container Registry page](#push-a-containerized-application-image-to-gitlab-container-registry). - -  - -1. Add a port mapping. Set `80` to **Host Port** and `5000` to **Container port**. - -  - -1. Select **Create**. - -Now you have the initial task definition. Next, you create an actual infrastructure to run the -application image. - -### Create an ECS cluster - -An [ECS cluster](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/clusters.html) -is a virtual group of [ECS services](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html). -It's also associated with EC2 or Fargate as the computation resource. - -1. Go to **ECS > Clusters** on [AWS console](https://aws.amazon.com/). -1. Select **Create Cluster**. -1. Select **EC2 Linux + Networking** as the cluster template. Select **Next Step**. -1. Set `ecs-demo` to **Cluster Name**. -1. Choose the default [VPC](https://aws.amazon.com/vpc/?vpc-blogs.sort-by=item.additionalFields.createdDate&vpc-blogs.sort-order=desc) - in **Networking**. If there are no existing VPCs, you can leave it as-is to create a new one. -1. Set all available subnets of the VPC to **Subnets**. -1. Select **Create**. -1. Make sure that the ECS cluster has been successfully created. - -  - -Now you can register an ECS service to the ECS cluster in the next step. - -Note the following: - -- Optionally, you can set a SSH key pair in the creation form. This allows you to SSH to the EC2 - instance for debugging. -- If you don't choose an existing VPC, it creates a new VPC by default. This could cause an error if - it reaches the maximum allowed number of internet gateways on your account. -- The cluster requires an EC2 instance, meaning it costs you [according to the instance-type](https://aws.amazon.com/ec2/pricing/on-demand/). - -### Create an ECS Service - -[ECS service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html) -is a daemon to create an application container based on the [ECS task definition](#create-an-ecs-task-definition). - -1. Go to **ECS > Clusters > ecs-demo > Services** on the [AWS console](https://aws.amazon.com/) -1. Select **Deploy**. This opens a service creation form. -1. Select `EC2` in **Launch Type**. -1. Set `ecs_demo` to **Task definition**. This corresponds to [the task definition you created above](#create-an-ecs-task-definition). -1. Set `ecs_demo` to **Service name**. -1. Set `1` to **Desired tasks**. - -  - -1. Select **Deploy**. -1. Make sure that the created service is active. - -  - -Note that AWS's console UI changes from time to time. If you can't find a relevant component in the -instructions, select the closest one. - -### View the demo application - -Now, the demo application is accessible from the internet. - -1. Go to **EC2 > Instances** on the [AWS console](https://aws.amazon.com/) -1. Search by `ECS Instance` to find the corresponding EC2 instance that [the ECS cluster created](#create-an-ecs-cluster). -1. Select the ID of the EC2 instance. This brings you to the instance detail page. -1. Copy **Public IPv4 address** and paste it in the browser. Now you can see the demo application - running. - -  - -In this guide, HTTPS/SSL is **NOT** configured. You can access to the application through HTTP only -(for example, `http://<ec2-ipv4-address>`). - -## Setup Continuous Deployment from GitLab - -Now that you have an application running on ECS, you can set up continuous deployment from GitLab. - -### Create a new IAM user as a deployer - -For GitLab to access the ECS cluster, service, and task definition that you created above, You must -create a deployer user on AWS: - -1. Go to **IAM > Users** on [AWS console](https://aws.amazon.com/). -1. Select **Add user**. -1. Set `ecs_demo` to **User name**. -1. Enable **Programmatic access** checkbox. Select **Next: Permissions**. -1. Select `Attach existing policies directly` in **Set permissions**. -1. Select `AmazonECS_FullAccess` from the policy list. Select **Next: Tags** and **Next: Review**. - -  - -1. Select **Create user**. -1. Take note of the **Access key ID** and **Secret access key** of the created user. - -NOTE: -Do not share the secret access key in a public place. You must save it in a secure place. - -### Setup credentials in GitLab to let pipeline jobs access to ECS - -You can register the access information in [GitLab Environment Variables](../../variables/index.md#custom-cicd-variables). -These variables are injected into the pipeline jobs and can access the ECS API. - -1. Go to **ecs-demo** project on GitLab. -1. Go to **Settings > CI/CD > Variables**. -1. Select **Add Variable** and set the following key-value pairs. - - |Key|Value|Note| - |---|---|---| - |`AWS_ACCESS_KEY_ID`|`<Access key ID of the deployer>`| For authenticating `aws` CLI. | - |`AWS_SECRET_ACCESS_KEY`|`<Secret access key of the deployer>`| For authenticating `aws` CLI. | - |`AWS_DEFAULT_REGION`|`us-east-2`| For authenticating `aws` CLI. | - |`CI_AWS_ECS_CLUSTER`|`ecs-demo`| The ECS cluster is accessed by `production_ecs` job. | - |`CI_AWS_ECS_SERVICE`|`ecs_demo`| The ECS service of the cluster is updated by `production_ecs` job. | - |`CI_AWS_ECS_TASK_DEFINITION`|`ecs_demo`| The ECS task definition is updated by `production_ecs` job. | - -### Make a change to the demo application - -Change a file in the project and see if it's reflected in the demo application on ECS: - -1. Go to **ecs-demo** project on GitLab. -1. Open the file at **app > views > welcome > `index.html.erb`**. -1. Select **Edit**. -1. Change the text to `You're on ECS!`. -1. Select **Commit Changes**. This automatically triggers a new pipeline. Wait until it finishes. -1. [Access the running application on the ECS cluster](#view-the-demo-application). You should see - this: - -  - -Congratulations! You successfully set up continuous deployment to ECS. - -## Further reading - -- If you're interested in more of the continuous deployments to clouds, see [cloud deployments](../index.md). -- If you want to quickly set up DevSecOps in your project, see [Auto DevOps](../../../topics/autodevops/index.md). -- If you want to quickly set up the production-grade environment, see [the 5 Minute Production App](https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/master/README.md). +<!-- This redirect file can be deleted after <2022-09-18>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index c89ce2675e4..c5be2328264 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -5,165 +5,101 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# Cloud deployment **(FREE)** +# Deploy to AWS from GitLab CI/CD **(FREE)** -Interacting with a major cloud provider may have become a much needed task that's -part of your delivery process. With GitLab you can -[deploy your application anywhere](https://about.gitlab.com/stages-devops-lifecycle/deploy-targets/). +GitLab provides Docker images with the libraries and tools you need to deploy +to AWS. You can reference these images in your CI/CD pipeline. -For some specific deployment targets, GitLab makes this process less painful by providing Docker -images with the needed libraries and tools pre-installed. By referencing them in your -CI/CD pipeline, you can interact with your chosen cloud provider more easily. +If you're using GitLab.com and deploying to the [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (ECS), +read about [deploying to ECS](ecs/quick_start_guide.md). -## AWS +## Authenticate GitLab with AWS -GitLab provides Docker images that you can use to [run AWS commands from GitLab CI/CD](#run-aws-commands-from-gitlab-cicd), and a template to make -it easier to [deploy to AWS](#deploy-your-application-to-the-aws-elastic-container-service-ecs). +To use GitLab CI/CD to connect to AWS, you must authenticate. +After you set up authentication, you can configure CI/CD to deploy. -### Quick start +1. Sign on to your AWS account. +1. Create [an IAM user](https://console.aws.amazon.com/iam/home#/home). +1. Select your user to access its details. Go to **Security credentials > Create a new access key**. +1. Note the **Access key ID** and **Secret access key**. +1. In your GitLab project, go to **Settings > CI/CD**. Set the following + [CI/CD variables](../variables/index.md): -If you're using GitLab.com, see the [quick start guide](ecs/quick_start_guide.md) -for setting up Continuous Deployment to [AWS Elastic Container Service](https://aws.amazon.com/ecs/) (ECS). + | Environment variable name | Value | + |:-------------------------------|:------------------------| + | `AWS_ACCESS_KEY_ID` | Your Access key ID. | + | `AWS_SECRET_ACCESS_KEY` | Your secret access key. | + | `AWS_DEFAULT_REGION` | Your region code. You might want to confirm that the AWS service you intend to use is [available in the chosen region](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/). | -### Run AWS commands from GitLab CI/CD +1. Variables are [protected by default](../variables/index.md#protected-cicd-variables). + To use GitLab CI/CD with branches or tags that are not protected, + clear the **Protect variable** checkbox. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31167) in GitLab 12.6. +## Use an image to run AWS commands -The GitLab AWS Docker image provides the [AWS Command Line Interface](https://aws.amazon.com/cli/), -which enables you to run `aws` commands. As part of your deployment strategy, you can run `aws` commands directly from -`.gitlab-ci.yml` by specifying the [GitLab AWS Docker image](https://gitlab.com/gitlab-org/cloud-deploy). +If an image contains the [AWS Command Line Interface](https://aws.amazon.com/cli/), +you can reference the image in your project's `.gitlab-ci.yml` file. Then you can run +`aws` commands in your CI/CD jobs. -Some credentials are required to be able to run `aws` commands: +For example: -1. Sign up for [an AWS account](https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-set-up.html) if you don't have one yet. -1. Log in onto the console and create [a new IAM user](https://console.aws.amazon.com/iam/home#/home). -1. Select your newly created user to access its details. Navigate to **Security credentials > Create a new access key**. - - NOTE: - A new **Access key ID** and **Secret access key** are generated. Please take a note of them right away. - -1. In your GitLab project, go to **Settings > CI/CD**. Set the following as - [CI/CD variables](../variables/index.md) - (see table below): - - - Access key ID. - - Secret access key. - - Region code. You can check the [list of AWS regional endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). - You might want to check if the AWS service you intend to use is - [available in the chosen region](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/). - - | Environment variable name | Value | - |:-------------------------------|:-----------------------| - | `AWS_ACCESS_KEY_ID` | Your Access key ID | - | `AWS_SECRET_ACCESS_KEY` | Your Secret access key | - | `AWS_DEFAULT_REGION` | Your region code | - - NOTE: - When you create a variable it's set to be [protected by default](../variables/index.md#protected-cicd-variables). If you want to use the `aws` commands on branches or tags that are not protected, make sure to uncheck the **Protect variable** checkbox. - -1. You can now use `aws` commands in the `.gitlab-ci.yml` file of this project: - - ```yaml - deploy: - stage: deploy - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest # see the note below - script: - - aws s3 ... - - aws create-deployment ... - ``` +```yaml +deploy: + stage: deploy + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + script: + - aws s3 ... + - aws create-deployment ... +``` - NOTE: - The image used in the example above - (`registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest`) is hosted on the [GitLab - Container Registry](../../user/packages/container_registry/index.md) and is - ready to use. Alternatively, replace the image with one hosted on AWS ECR. +GitLab provides a Docker image that includes the AWS CLI: -### Use an AWS Elastic Container Registry (ECR) image in your CI/CD +- Images are hosted in the GitLab Container Registry. The latest image is + `registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest`. +- [Images are stored in a GitLab repository](https://gitlab.com/gitlab-org/cloud-deploy/-/tree/master/aws). -Instead of referencing an image hosted on the GitLab Registry, you can -reference an image hosted on any third-party registry, such as the -[Amazon Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/). +Alternately, you can use an [Amazon Elastic Container Registry (ECR)](https://aws.amazon.com/ecr/) image. +[Learn how to push an image to your ECR repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). -To do so, [push your image into your ECR -repository](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). -Then reference it in your `.gitlab-ci.yml` file and replace the `image` -path to point to your ECR image. +You can also use an image from any third-party registry. -### Deploy your application to the AWS Elastic Container Service (ECS) +## Deploy your application to ECS > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207962) in GitLab 12.9. > - The `Deploy-ECS.gitlab-ci.yml` template was [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/220821) to `AWS/Deploy-ECS.gitlab-ci.yml` in GitLab 13.2. -GitLab provides a series of [CI templates that you can include in your project](../yaml/index.md#include). -To automate deployments of your application to your [Amazon Elastic Container Service](https://aws.amazon.com/ecs/) (AWS ECS) -cluster, you can `include` the `AWS/Deploy-ECS.gitlab-ci.yml` template in your `.gitlab-ci.yml` file. +You can automate deployments of your application to your [Amazon ECS](https://aws.amazon.com/ecs/) +cluster. -GitLab also provides [Docker images](https://gitlab.com/gitlab-org/cloud-deploy/-/tree/master/aws) that can be used in your `.gitlab-ci.yml` file to simplify working with AWS: +Prerequisites: -- Use `registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest` to use AWS CLI commands. -- Use `registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest` to deploy your application to AWS ECS. +- [Authenticate AWS with GitLab](#authenticate-gitlab-with-aws). +- Create a cluster on Amazon ECS. +- Create related components, like an ECS service, a database on Amazon RDS, and so on. +- Create an ECS task definition, where the value for the `containerDefinitions[].name` attribute is + the same as the `Container name` defined in your targeted ECS service. The task definition can be: + - An existing task definition in ECS. + - [In GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/222618), + a JSON file in your GitLab project. Use the + [template in the AWS documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-task-definition.html#task-definition-template) + and save the file in your project. For example `<project-root>/ci/aws/task-definition.json`. -Before getting started with this process, you need a cluster on AWS ECS, as well as related -components, like an ECS service, ECS task definition, a database on Amazon RDS, and so on. -[Read more about AWS ECS](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/Welcome.html). - -The ECS task definition can be: - -- An existing task definition in AWS ECS -- A JSON file containing a task definition. Create the JSON file by using the template provided in - the [AWS documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-task-definition.html#task-definition-template). - Copy the task definition into a new file in your project, for example `<project-root>/ci/aws/task-definition.json`. - [Available](https://gitlab.com/gitlab-org/gitlab/-/issues/222618) in GitLab 13.3 and later. - -After you have these prerequisites ready, follow these steps: - -1. Make sure your AWS credentials are set up as CI/CD variables for your - project. You can follow [the steps above](#run-aws-commands-from-gitlab-cicd) to complete this setup. -1. Add these variables to your project's `.gitlab-ci.yml` file, or in the project's - [CI/CD settings](../variables/index.md#custom-cicd-variables): - - - `CI_AWS_ECS_CLUSTER`: The name of the AWS ECS cluster that you're targeting for your deployments. - - `CI_AWS_ECS_SERVICE`: The name of the targeted service tied to your AWS ECS cluster. - - `CI_AWS_ECS_TASK_DEFINITION`: The name of an existing task definition in ECS tied - to the service mentioned above. - - ```yaml - variables: - CI_AWS_ECS_CLUSTER: my-cluster - CI_AWS_ECS_SERVICE: my-service - CI_AWS_ECS_TASK_DEFINITION: my-task-definition - ``` - - You can find these names after selecting the targeted cluster on your [AWS ECS dashboard](https://console.aws.amazon.com/ecs/home): - -  - - Alternatively, if you want to use a task definition defined in a JSON file, use - `CI_AWS_ECS_TASK_DEFINITION_FILE` instead: - - ```yaml - variables: - CI_AWS_ECS_CLUSTER: my-cluster - CI_AWS_ECS_SERVICE: my-service - CI_AWS_ECS_TASK_DEFINITION_FILE: ci/aws/my_task_definition.json - ``` +To deploy to your ECS cluster: - You can create your `CI_AWS_ECS_TASK_DEFINITION_FILE` variable as a - [file-typed CI/CD variable](../variables/index.md#cicd-variable-types) instead of a - regular CI/CD variable. If you choose to do so, set the variable value to be the full contents of - the JSON task definition. You can then remove the JSON file from your project. +1. In your GitLab project, go to **Settings > CI/CD**. Set the following + [CI/CD variables](../variables/index.md). You can find these names by + selecting the targeted cluster on your [Amazon ECS dashboard](https://console.aws.amazon.com/ecs/home). - In both cases, make sure that the value for the `containerDefinitions[].name` attribute is - the same as the `Container name` defined in your targeted ECS service. + | Environment variable name | Value | + |:-------------------------------|:------------------------| + | `CI_AWS_ECS_CLUSTER` | The name of the AWS ECS cluster that you're targeting for your deployments. | + | `CI_AWS_ECS_SERVICE` | The name of the targeted service tied to your AWS ECS cluster. | + | `CI_AWS_ECS_TASK_DEFINITION` | If the task definition is in ECS, the name of the task definition tied to the service. | + | `CI_AWS_ECS_TASK_DEFINITION_FILE` | If the task definition is a JSON file in GitLab, the filename, including the path. For example, `ci/aws/my_task_definition.json`. If the name of the task definition in your JSON file is the same name as an existing task definition in ECS, then a new revision is created when CI/CD runs. Otherwise, a brand new task definition is created, starting at revision 1. | WARNING: - `CI_AWS_ECS_TASK_DEFINITION_FILE` takes precedence over `CI_AWS_ECS_TASK_DEFINITION` if both these - variables are defined within your project. - - NOTE: - If the name of the task definition you wrote in your JSON file is the same name - as an existing task definition on AWS, then a new revision is created for it. - Otherwise, a brand new task definition is created, starting at revision 1. + If you define both `CI_AWS_ECS_TASK_DEFINITION_FILE` and `CI_AWS_ECS_TASK_DEFINITION`, + `CI_AWS_ECS_TASK_DEFINITION_FILE` takes precedence. 1. Include this template in `.gitlab-ci.yml`: @@ -172,86 +108,78 @@ After you have these prerequisites ready, follow these steps: - template: AWS/Deploy-ECS.gitlab-ci.yml ``` - The `AWS/Deploy-ECS` template ships with GitLab and is available [on - GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml). + The `AWS/Deploy-ECS` template ships with GitLab and is available + [on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml). -1. Commit and push your updated `.gitlab-ci.yml` to your project's repository, and you're done! +1. Commit and push your updated `.gitlab-ci.yml` to your project's repository. - Your application Docker image is rebuilt and pushed to the GitLab registry. - If your image is located in a private registry, make sure your task definition is - [configured with a `repositoryCredentials` attribute](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html). +Your application Docker image is rebuilt and pushed to the GitLab Container Registry. +If your image is located in a private registry, make sure your task definition is +[configured with a `repositoryCredentials` attribute](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html). - Then the targeted task definition is updated with the location of the new - Docker image, and a new revision is created in ECS as result. +The targeted task definition is updated with the location of the new +Docker image, and a new revision is created in ECS as result. - Finally, your AWS ECS service is updated with the new revision of the - task definition, making the cluster pull the newest version of your - application. +Finally, your AWS ECS service is updated with the new revision of the +task definition, making the cluster pull the newest version of your +application. + +NOTE: +ECS deploy jobs wait for the rollout to complete before exiting. To disable this behavior, +set `CI_AWS_ECS_WAIT_FOR_ROLLOUT_COMPLETE_DISABLED` to a non-empty value. WARNING: The [`AWS/Deploy-ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml) -template includes both the [`Jobs/Build.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml) -and [`Jobs/Deploy/ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml) -"sub-templates". Do not include these "sub-templates" on their own, and only include the main -`AWS/Deploy-ECS.gitlab-ci.yml` template. The "sub-templates" are designed to only be -used along with the main template. They may move or change unexpectedly causing your -pipeline to fail if you didn't include the main template. Also, the job names within -these templates may change. Do not override these jobs names in your own pipeline, -as the override stops working when the name changes. - -Alternatively, if you don't wish to use the `AWS/Deploy-ECS.gitlab-ci.yml` template -to deploy to AWS ECS, you can always use our -`aws-base` Docker image to run your own [AWS CLI commands for ECS](https://docs.aws.amazon.com/cli/latest/reference/ecs/index.html#cli-aws-ecs). +template includes two templates: [`Jobs/Build.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml) +and [`Jobs/Deploy/ECS.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml). Do not include these templates on their own. Only include the +`AWS/Deploy-ECS.gitlab-ci.yml` template. These other templates are designed to be +used only with the main template. They may move or change unexpectedly. Also, the job names within +these templates may change. Do not override these job names in your own pipeline, +because the override stops working when the name changes. -```yaml -deploy: - stage: deploy - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - script: - - aws ecs register-task-definition ... -``` - -### Provision and deploy to your AWS Elastic Compute Cloud (EC2) +## Deploy your application to EC2 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201742) in GitLab 13.5. -You can use the `AWS/CF-Provision-and-Deploy-EC2` CI template to perform the -following actions within the same pipeline: +GitLab provides a template, called `AWS/CF-Provision-and-Deploy-EC2`, +to assist you in deploying to Amazon EC2. -1. **Create stack**: Provision your own infrastructure by leveraging the [AWS CloudFormation](https://aws.amazon.com/cloudformation/) API. -1. **Push to S3**: Push your previously-built artifact to an [AWS S3](https://aws.amazon.com/s3/) bucket. -1. **Deploy to EC2**: Deploy this pushed content onto an [AWS EC2](https://aws.amazon.com/ec2/) instance. +When you configure related JSON objects and use the template, the pipeline: - +1. **Creates the stack**: Your infrastructure is provisioned by using + the [AWS CloudFormation](https://aws.amazon.com/cloudformation/) API. +1. **Pushes to an S3 bucket**: When your build runs, it creates an artifact. + The artifact is pushed to an [AWS S3](https://aws.amazon.com/s3/) bucket. +1. **Deploys to EC2**: The content is deployed on an [AWS EC2](https://aws.amazon.com/ec2/) instance. -#### Run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template + -To run the `AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml` template, you must -pass three JSON input objects, based on existing templates: +### Configure the template and JSON -1. The AWS documentation provides templates for the _Create stack_ and _Deploy to EC2_ steps (links - below). We provide the template for the remaining step, _Push to S3_: +To deploy to EC2, complete the following steps. - - [Template for the _Create stack_ step on AWS](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-anatomy.html). - - Template for the _Push to S3_ step. Note that `source` is where a preceding `build` job built - your application, exporting the build through [`artifacts:paths`](../yaml/index.md#artifactspaths): +1. Create JSON for your stack. Use the [AWS template](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-anatomy.html). +1. Create JSON to push to S3. Include the following details. - ```json - { - "applicationName": "string", - "source": "string", - "s3Location": "s3://your/bucket/project_built_file...]" - } - ``` + ```json + { + "applicationName": "string", + "source": "string", + "s3Location": "s3://your/bucket/project_built_file...]" + } + ``` - - [Template for the _Deploy to EC2_ step on AWS](https://docs.aws.amazon.com/codedeploy/latest/APIReference/API_CreateDeployment.html). + The `source` is the location where a `build` job built your application. + The build is saved to [`artifacts:paths`](../yaml/index.md#artifactspaths). -1. After you have completed these three templates based on your requirements, you - have two ways to pass in these JSON objects: +1. Create JSON to deploy to EC2. Use the [AWS template](https://docs.aws.amazon.com/codedeploy/latest/APIReference/API_CreateDeployment.html). +1. Make the JSON objects accessible to your pipeline: + - If you want these JSON objects saved in your repository, save the objects as three + separate files. - - They can be three actual files located in your project. You must specify their path relative to - your project root in your `.gitlab-ci.yml` file, using the following CI/CD variables. For example, if - your files are in a `<project_root>/aws` folder: + In your `.gitlab-ci.yml` file, add [CI/CD variables](../variables/index.md) + that point to the file paths relative to the project root. For example, + if your JSON files are in a `<project_root>/aws` folder: ```yaml variables: @@ -260,71 +188,30 @@ pass three JSON input objects, based on existing templates: CI_AWS_EC2_DEPLOYMENT_FILE: 'aws/create_deployment.json' ``` - - Alternatively, you can provide these JSON objects as [file-typed CI/CD variables](../variables/index.md#cicd-variable-types). - In your project, go to **Settings > CI/CD > Variables** and add - the three variables listed above as file-typed CI/CD variables. - For each variable, set the value to its corresponding JSON object. + - If you do not want these JSON objects saved in your repository, add each object + as a separate [file type CI/CD variable](../variables/index.md#cicd-variable-types) + in the project settings. Use the same variable names as above. -1. Provide the name of the stack you're creating and/or targeting, as a CI/CD variable: +1. In your `.gitlab-ci.yml` file, create a CI/CD variable for the name of the stack. For example: ```yaml variables: CI_AWS_CF_STACK_NAME: 'YourStackName' ``` -1. Add this CI template to your `.gitlab-ci.yml`: +1. In your `.gitlab-ci.yml` file, add the CI template: ```yaml include: - template: AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml ``` -When running your project pipeline at this point: - -- Your AWS CloudFormation stack is created based on the content of your - `CI_AWS_CF_CREATE_STACK_FILE` file/variable. - If your stack already exists, this step is skipped, but the `provision` job it belongs to still - runs. -- Your built application is pushed to your S3 bucket then and deployed to your EC2 instance, based - on the related JSON object's content. The deployment job finishes whenever the deployment to EC2 - is done or has failed. - -#### Custom build job for Auto DevOps - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216008) in GitLab 13.6. - -To leverage [Auto DevOps](../../topics/autodevops/index.md) for your project when deploying to -AWS EC2, first you must define [your AWS credentials as CI/CD variables](#run-aws-commands-from-gitlab-cicd). - -Next, define a job for the `build` stage. To do so, you must reference the -`Auto-DevOps.gitlab-ci.yml` template and include a job named `build_artifact` in your -`.gitlab-ci.yml` file. For example: - -```yaml -# .gitlab-ci.yml - -include: - - template: Auto-DevOps.gitlab-ci.yml - -variables: - AUTO_DEVOPS_PLATFORM_TARGET: EC2 - -build_artifact: - stage: build - script: - - <your build script goes here> - artifacts: - paths: - - <built artifact> -``` - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For a video walkthrough of this configuration process, see [Auto Deploy to EC2](https://www.youtube.com/watch?v=4B-qSwKnacA). - -### Deploy to Amazon EKS - -- [How to deploy your application to a GitLab-managed Amazon EKS cluster with Auto DevOps](https://about.gitlab.com/blog/2020/05/05/deploying-application-eks/) - -## Deploy to Google Cloud +1. Run the pipeline. -- [Deploying with GitLab on Google Cloud](https://about.gitlab.com/partners/technology-partners/google-cloud-platform/) + - Your AWS CloudFormation stack is created based on the content of your + `CI_AWS_CF_CREATE_STACK_FILE` variable. + If your stack already exists, this step is skipped, but the `provision` + job it belongs to still runs. + - Your built application is pushed to your S3 bucket then and deployed to your EC2 instance, based + on the related JSON object's content. The deployment job finishes when the deployment to EC2 + is done or has failed. diff --git a/doc/ci/cloud_services/google_cloud/index.md b/doc/ci/cloud_services/google_cloud/index.md index 14928f91816..f4e4a2046ba 100644 --- a/doc/ci/cloud_services/google_cloud/index.md +++ b/doc/ci/cloud_services/google_cloud/index.md @@ -50,8 +50,8 @@ inside the Workload Identity Pool created in the previous step, using the follow such as `gitlab/gitlab`. - **Provider ID**: Unique ID in the pool for the Workload Identity Provider, such as `gitlab-gitlab`. This value is used to refer to the provider, and appears in URLs. -- **Issuer (URL)**: The address of your GitLab instance, such as `https://gitlab.com` or - `https://gitlab.example.com`. +- **Issuer (URL)**: The address of your GitLab instance, such as `https://gitlab.com/` or + `https://gitlab.example.com/`. - The address must use the `https://` protocol. - The address must end in a trailing slash. - **Audiences**: Manually set the allowed audiences list to the address of your diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index 5d0aaf7cc45..f63ecc57028 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -87,7 +87,7 @@ are certain use cases that you may need to work around. For more information, ch The needs visualization makes it easier to visualize the relationships between dependent jobs in a DAG. This graph displays all the jobs in a pipeline that need or are needed by other jobs. Jobs with no relationships are not displayed in this view. -To see the needs visualization, click on the **Needs** tab when viewing a pipeline that uses the `needs` keyword. +To see the needs visualization, select **Needs** when viewing a pipeline that uses the `needs` keyword.  diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 9b91cd40338..df0c7b69d46 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -92,7 +92,7 @@ the job script in context of the image in privileged mode. We recommend you use Docker-in-Docker with TLS enabled, which is supported by [GitLab.com shared runners](../runners/index.md). -You should always specify a specific version of the image, like `docker:19.03.12`. +You should always specify a specific version of the image, like `docker:20.10.16`. If you use a tag like `docker:stable`, you have no control over which version is used. Unpredictable behavior can result, especially when new versions are released. @@ -126,12 +126,12 @@ To use Docker-in-Docker with TLS enabled: --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.12" \ + --docker-image "docker:20.10.16" \ --docker-privileged \ --docker-volumes "/certs/client" ``` - - This command registers a new runner to use the `docker:19.03.12` image. + - This command registers a new runner to use the `docker:20.10.16` image. To start the build and service containers, it uses the `privileged` mode. If you want to use [Docker-in-Docker](https://www.docker.com/blog/docker-can-now-run-within-docker/), you must always use `privileged = true` in your Docker containers. @@ -149,7 +149,7 @@ To use Docker-in-Docker with TLS enabled: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.12" + image = "docker:20.10.16" privileged = true disable_cache = false volumes = ["/certs/client", "/cache"] @@ -159,10 +159,10 @@ To use Docker-in-Docker with TLS enabled: ``` 1. You can now use `docker` in the job script. Note the inclusion of the - `docker:19.03.12-dind` service: + `docker:20.10.16-dind` service: ```yaml - image: docker:19.03.12 + image: docker:20.10.16 variables: # When you use the dind service, you must instruct Docker to talk with @@ -182,7 +182,7 @@ To use Docker-in-Docker with TLS enabled: DOCKER_TLS_CERTDIR: "/certs" services: - - docker:19.03.12-dind + - docker:20.10.16-dind before_script: - docker info @@ -209,7 +209,7 @@ Assuming that the runner's `config.toml` is similar to: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.12" + image = "docker:20.10.16" privileged = true disable_cache = false volumes = ["/cache"] @@ -219,10 +219,10 @@ Assuming that the runner's `config.toml` is similar to: ``` You can now use `docker` in the job script. Note the inclusion of the -`docker:19.03.12-dind` service: +`docker:20.10.16-dind` service: ```yaml -image: docker:19.03.12 +image: docker:20.10.16 variables: # When using dind service, you must instruct docker to talk with the @@ -243,7 +243,7 @@ variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - docker:20.10.16-dind before_script: - docker info @@ -284,10 +284,10 @@ To use Docker-in-Docker with TLS enabled in Kubernetes: ``` 1. You can now use `docker` in the job script. Note the inclusion of the - `docker:19.03.13-dind` service: + `docker:20.10.16-dind` service: ```yaml - image: docker:19.03.13 + image: docker:20.10.16 variables: # When using dind service, you must instruct Docker to talk with @@ -315,7 +315,7 @@ To use Docker-in-Docker with TLS enabled in Kubernetes: DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" services: - - docker:19.03.13-dind + - docker:20.10.16-dind before_script: - docker info @@ -341,7 +341,7 @@ not without its own challenges: - **Storage drivers**: By default, earlier versions of Docker use the `vfs` storage driver, which copies the file system for each job. Docker 17.09 and later use `--storage-driver overlay2`, which is the recommended storage driver. See [Using the OverlayFS driver](#use-the-overlayfs-driver) for details. -- **Root file system**: Because the `docker:19.03.12-dind` container and the runner container don't share their +- **Root file system**: Because the `docker:20.10.16-dind` container and the runner container don't share their root file system, you can use the job's working directory as a mount point for child containers. For example, if you have files you want to share with a child container, you might create a subdirectory under `/builds/$CI_PROJECT_PATH` @@ -364,7 +364,7 @@ container. Docker is then available in the context of the image. NOTE: If you bind the Docker socket and you are [using GitLab Runner 11.11 or later](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1261), -you can no longer use `docker:19.03.12-dind` as a service. Volume bindings +you can no longer use `docker:20.10.16-dind` as a service. Volume bindings are done to the services as well, making these incompatible. #### Use the Docker executor with Docker socket binding @@ -383,7 +383,7 @@ Your configuration should look something like this: executor = "docker" [runners.docker] tls_verify = false - image = "docker:19.03.12" + image = "docker:20.10.16" privileged = false disable_cache = false volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/cache"] @@ -399,7 +399,7 @@ sudo gitlab-runner register -n \ --registration-token REGISTRATION_TOKEN \ --executor docker \ --description "My Docker Runner" \ - --docker-image "docker:19.03.12" \ + --docker-image "docker:20.10.16" \ --docker-volumes /var/run/docker.sock:/var/run/docker.sock ``` @@ -417,7 +417,7 @@ mirror: ```yaml services: - - name: docker:19.03.13-dind + - name: docker:20.10.16-dind command: ["--registry-mirror", "https://registry-mirror.example.com"] # Specify the registry mirror to use ``` @@ -440,7 +440,7 @@ Docker: ... privileged = true [[runners.docker.services]] - name = "docker:19.03.13-dind" + name = "docker:20.10.16-dind" command = ["--registry-mirror", "https://registry-mirror.example.com"] ``` @@ -454,7 +454,7 @@ Kubernetes: ... privileged = true [[runners.kubernetes.services]] - name = "docker:19.03.13-dind" + name = "docker:20.10.16-dind" command = ["--registry-mirror", "https://registry-mirror.example.com"] ``` @@ -563,11 +563,11 @@ the implications of this method are: docker run --rm -t -i -v $(pwd)/src:/home/app/src test-image:latest run_app_tests ``` -You don't need to include the `docker:19.03.12-dind` service, like you do when +You don't need to include the `docker:20.10.16-dind` service, like you do when you're using the Docker-in-Docker executor: ```yaml -image: docker:19.03.12 +image: docker:20.10.16 before_script: - docker info @@ -591,13 +591,13 @@ In [`before_script`](../yaml/index.md#before_script), run `docker login`: ```yaml -image: docker:19.03.13 +image: docker:20.10.16 variables: DOCKER_TLS_CERTDIR: "/certs" services: - - docker:19.03.13-dind + - docker:20.10.16-dind build: stage: build @@ -616,7 +616,7 @@ empty or remove it. If you are an administrator for GitLab Runner, you can mount a file with the authentication configuration to `~/.docker/config.json`. Then every job that the runner picks up is authenticated already. If you -are using the official `docker:19.03.13` image, the home directory is +are using the official `docker:20.10.16` image, the home directory is under `/root`. If you mount the configuration file, any `docker` command @@ -699,13 +699,13 @@ The following example shows [`before_script`](../yaml/index.md#before_script). The same commands apply for any solution you implement. ```yaml -image: docker:19.03.13 +image: docker:20.10.16 variables: DOCKER_TLS_CERTDIR: "/certs" services: - - docker:19.03.13-dind + - docker:20.10.16-dind build: stage: build @@ -741,10 +741,10 @@ with the `--cache-from` argument must first be pulled Here's a `.gitlab-ci.yml` file that shows how to use Docker caching: ```yaml -image: docker:19.03.12 +image: docker:20.10.16 services: - - docker:19.03.12-dind + - docker:20.10.16-dind variables: # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled @@ -862,10 +862,10 @@ This issue can occur when the service's image name [includes a registry hostname](../../ci/services/index.md#available-settings-for-services). For example: ```yaml -image: docker:19.03.12 +image: docker:20.10.16 services: - - registry.hub.docker.com/library/docker:19.03.12-dind + - registry.hub.docker.com/library/docker:20.10.16-dind ``` A service's hostname is [derived from the full image name](../../ci/services/index.md#accessing-the-services). @@ -873,9 +873,9 @@ However, the shorter service hostname `docker` is expected. To allow service resolution and access, add an explicit alias for the service name `docker`: ```yaml -image: docker:19.03.12 +image: docker:20.10.16 services: - - name: registry.hub.docker.com/library/docker:19.03.12-dind + - name: registry.hub.docker.com/library/docker:20.10.16-dind alias: docker ``` diff --git a/doc/ci/docker/using_docker_images.md b/doc/ci/docker/using_docker_images.md index 7edff334134..fdd8b6d38b8 100644 --- a/doc/ci/docker/using_docker_images.md +++ b/doc/ci/docker/using_docker_images.md @@ -112,7 +112,7 @@ For example, the following two definitions are equal: image: "registry.example.com/my/image:latest" services: - - postgresql:9.4 + - postgresql:14.3 - redis:latest ``` @@ -124,7 +124,7 @@ For example, the following two definitions are equal: name: "registry.example.com/my/image:latest" services: - - name: postgresql:9.4 + - name: postgresql:14.3 - name: redis:latest ``` diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index 2e514fd0f0a..dac52a4540e 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -5,50 +5,46 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: howto --- -# How to enable or disable GitLab CI/CD **(FREE)** - -To use GitLab CI/CD, you need: - -- A valid [`.gitlab-ci.yml`](yaml/index.md) file present at the root directory - of your project. -- A [runner](runners/index.md) ready to run jobs. - -You can read our [quick start guide](quick_start/index.md) to get you started. +# Disabling GitLab CI/CD **(FREE)** +GitLab CI/CD is enabled by default on all new projects. If you use an external CI/CD server like Jenkins or Drone CI, you can disable GitLab CI/CD to avoid conflicts with the commits status API. -GitLab CI/CD is enabled by default on all new projects. You can: +You can disable GitLab CI/CD: -- Disable GitLab CI/CD [under each project's settings](#enable-cicd-in-a-project). -- Set GitLab CI/CD to be [disabled in all new projects on an instance](../administration/cicd.md). +- [For each project](#disable-cicd-in-a-project). +- [For all new projects on an instance](../administration/cicd.md). -If you disable GitLab CI/CD in a project: +These changes do not apply to projects in an +[external integration](../user/project/integrations/index.md#available-integrations). -- The **CI/CD** item in the left sidebar is removed. -- The `/pipelines` and `/jobs` pages are no longer available. -- Existing jobs and pipelines are not deleted. Re-enable CI/CD to access them again. +## Disable CI/CD in a project -The project or instance settings do not enable or disable pipelines run in an -[external integration](../user/project/integrations/index.md#available-integrations). +When you disable GitLab CI/CD: -## Enable CI/CD in a project +- The **CI/CD** item in the left sidebar is removed. +- The `/pipelines` and `/jobs` pages are no longer available. +- Existing jobs and pipelines are hidden, not removed. -To enable or disable GitLab CI/CD pipelines in your project: +To disable GitLab CI/CD in your project: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > General**. 1. Expand **Visibility, project features, permissions**. -1. In the **Repository** section, turn on or off **CI/CD** as required. +1. In the **Repository** section, turn off **CI/CD**. +1. Select **Save changes**. -**Project visibility** also affects pipeline visibility. If set to: +## Enable CI/CD in a project -- **Private**: Only project members can access pipelines. -- **Internal** or **Public**: Pipelines can be set to either **Only Project Members** - or **Everyone With Access** by using the dropdown box. +To enable GitLab CI/CD in your project: -Press **Save changes** for the settings to take effect. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. In the **Repository** section, turn on **CI/CD**. +1. Select **Save changes**. <!-- ## Troubleshooting diff --git a/doc/ci/environments/environments_dashboard.md b/doc/ci/environments/environments_dashboard.md index 8ff31827ae7..c3c1e7868fd 100644 --- a/doc/ci/environments/environments_dashboard.md +++ b/doc/ci/environments/environments_dashboard.md @@ -36,9 +36,9 @@ environments are not displayed. To add a project to the dashboard: -1. Click the **Add projects** button in the home screen of the dashboard. +1. Select **Add projects** in the home screen of the dashboard. 1. Search and add one or more projects using the **Search your projects** field. -1. Click the **Add projects** button. +1. Select **Add projects**. Once added, you can see a summary of each project's environment operational health, including the latest commit, pipeline status, and deployment time. diff --git a/doc/ci/environments/incremental_rollouts.md b/doc/ci/environments/incremental_rollouts.md index d359ed078ff..ee2e708f822 100644 --- a/doc/ci/environments/incremental_rollouts.md +++ b/doc/ci/environments/incremental_rollouts.md @@ -63,7 +63,7 @@ rollout 10%: ROLLOUT_PERCENTAGE: 10 ``` -When the jobs are built, a **play** button appears next to the job's name. Click the **play** button +When the jobs are built, a **play** button appears next to the job's name. Select **play** to release each stage of pods. You can also rollback by running a lower percentage job. Once 100% is reached, you cannot roll back using this method. It is still possible to roll back by redeploying the old version using the **Rollback** button on the environment page. diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 7e1ef5efaa5..b13bd041d46 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -646,7 +646,7 @@ To delete a stopped environment in the GitLab UI: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208655) in GitLab 13.2. -You can define a job that accesses an environment for various purposes, such as verification or preparation. This +You can define a job that accesses an environment for various purposes, such as verification or preparation. This effectively bypasses deployment creation, so that you can adjust your CD workflow more accurately. To do so, add either `action: prepare`, `action: verify`, or `action: access` to the `environment` section of your job: diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index adc215c7aa1..35ee4f9dd33 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -127,20 +127,16 @@ they have the following privileges: ## Deployment-only access to protected environments Users granted access to a protected environment, but not push or merge access -to the branch deployed to it, are only granted access to deploy the environment. An individual in a -group with the Reporter role, or in groups added to the project with the Reporter -role, appears in the dropdown menu for deployment-only access. +to the branch deployed to it, are only granted access to deploy the environment. +[Invited groups](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group-of-users) added +to the project with [Reporter role](../../user/permissions.md#project-members-permissions), appear in the dropdown menu for deployment-only access. To add deployment-only access: -1. Add a group with the Reporter role. -1. Add users to the group. -1. Invite the group to be a project member. +1. Create a group with members who are granted to access to the protected environment, if it doesn't exist yet. +1. [Invite the group](../../user/project/members/share_project_with_groups.md#share-a-project-with-a-group-of-users) to the project with the Reporter role. 1. Follow the steps in [Protecting Environments](#protecting-environments). -Note that deployment-only access is the only possible access level for groups with the Reporter -role. - ## Modifying and unprotecting environments Maintainers can: @@ -208,7 +204,7 @@ configured: deployment ruleset. - Developers should be given no more than the Developer role for the top-level group, or explicitly given the Maintainer role for a child project - They do *NOT* have access to the CI/CD configurations in the + They do *not* have access to the CI/CD configurations in the top-level group, so operators can ensure that the critical configuration won't be accidentally changed by the developers. - For sub-groups and child projects: @@ -232,17 +228,23 @@ Having this configuration in place: ### Protect critical environments under a group -To protect a group-level environment: +To protect a group-level environment, make sure your environments have the correct +[`deployment_tier`](index.md#deployment-tier-of-environments) defined in `.gitlab-ci.yml`. -1. Make sure your environments have the correct - [`deployment_tier`](index.md#deployment-tier-of-environments) defined in - `.gitlab-ci.yml`. -1. Configure the group-level protected environments by using the - [REST API](../../api/group_protected_environments.md). +#### Using the UI -NOTE: -Configuration [with the UI](https://gitlab.com/gitlab-org/gitlab/-/issues/325249) -is scheduled for a later release. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325249) in GitLab 15.1. + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Protected environments**. +1. From the **Environment** list, select the [deployment tier of environments](index.md#deployment-tier-of-environments) you want to protect. +1. In the **Allowed to deploy** list, select the [subgroups](../../user/group/subgroups/index.md) you want to give deploy access to. +1. Select **Protect**. + +#### Using the API + +Configure the group-level protected environments by using the [REST API](../../api/group_protected_environments.md). ## Deployment approvals diff --git a/doc/ci/img/ecs_dashboard_v12_9.png b/doc/ci/img/ecs_dashboard_v12_9.png Binary files differdeleted file mode 100644 index 62b85c75ec2..00000000000 --- a/doc/ci/img/ecs_dashboard_v12_9.png +++ /dev/null diff --git a/doc/ci/index.md b/doc/ci/index.md index 05e97613384..4fe2dee32bf 100644 --- a/doc/ci/index.md +++ b/doc/ci/index.md @@ -89,7 +89,7 @@ GitLab CI/CD features, grouped by DevOps stage, include: | [GitLab CI/CD for external repositories](ci_cd_for_external_repos/index.md) | Get the benefits of GitLab CI/CD combined with repositories in GitHub and Bitbucket Cloud. | | [Interactive Web Terminals](interactive_web_terminal/index.md) | Open an interactive web terminal to debug the running jobs. | | [Review Apps](review_apps/index.md) | Configure GitLab CI/CD to preview code changes. | -| [Unit test reports](unit_test_reports.md) | Identify test failures directly on merge requests. | +| [Unit test reports](testing/unit_test_reports.md) | Identify test failures directly on merge requests. | | [Using Docker images](docker/using_docker_images.md) | Use GitLab and GitLab Runner with Docker to build and test applications. | | **Release** | | | [Auto Deploy](../topics/autodevops/stages.md#auto-deploy) | Deploy your application to a production environment in a Kubernetes cluster. | diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index 8e5c8c8ab45..9567bc9cd65 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -20,7 +20,7 @@ You can use a GitLab CI/CD job token to authenticate with specific API endpoints - [Get job artifacts](../../api/job_artifacts.md#get-job-artifacts). - [Get job token's job](../../api/jobs.md#get-job-tokens-job). - [Pipeline triggers](../../api/pipeline_triggers.md), using the `token=` parameter. -- [Releases](../../api/releases/index.md). +- [Releases](../../api/releases/index.md) and [Release links](../../api/releases/links.md). - [Terraform plan](../../user/infrastructure/index.md). NOTE: @@ -28,7 +28,7 @@ There's an open issue, [GitLab-#333444](https://gitlab.com/gitlab-org/gitlab/-/issues/333444), which prevents you from using a job token with internal projects. This bug only impacts self-managed GitLab instances. - + The token has the same permissions to access the API as the user that caused the job to run. A user can cause a job to run by pushing a commit, triggering a manual job, being the owner of a scheduled pipeline, and so on. Therefore, this user must be assigned to @@ -104,13 +104,12 @@ The job token scope is only for controlling access to private projects. There is [a proposal](https://gitlab.com/groups/gitlab-org/-/epics/3559) to improve the feature with more strategic control of the access permissions. -## Trigger a multi-project pipeline by using a CI job token +## Trigger a multi-project pipeline by using a CI/CD job token > `CI_JOB_TOKEN` for multi-project pipelines was [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/31573) from GitLab Premium to GitLab Free in 12.4. -You can use the `CI_JOB_TOKEN` to trigger [multi-project pipelines](../pipelines/multi_project_pipelines.md) -from a CI/CD job. A pipeline triggered this way creates a dependent pipeline relation -that is visible on the [pipeline graph](../pipelines/multi_project_pipelines.md#multi-project-pipeline-visualization). +You can use the `CI_JOB_TOKEN` to [trigger multi-project pipelines](../../api/pipeline_triggers.md#trigger-a-pipeline-with-a-token) +from a CI/CD job. For example: diff --git a/doc/ci/jobs/index.md b/doc/ci/jobs/index.md index 42827dc2d48..1f7a7436c0a 100644 --- a/doc/ci/jobs/index.md +++ b/doc/ci/jobs/index.md @@ -129,7 +129,7 @@ they are collapsed into a single group in regular pipeline graphs (not the mini You can recognize when a pipeline has grouped jobs if you don't see the retry or cancel button inside them. Hovering over them shows the number of grouped -jobs. Click to expand them. +jobs. Select to expand them.  @@ -266,7 +266,7 @@ In this example: When running manual jobs you can supply additional job specific variables. You can do this from the job page of the manual job you want to run with -additional variables. To access this page, click on the **name** of the manual job in +additional variables. To access this page, select the **name** of the manual job in the pipeline view, *not* the play (**{play}**) button. This is useful when you want to alter the execution of a job that uses @@ -328,7 +328,7 @@ job1: - echo -e "\e[0Ksection_end:`date +%s`:my_first_section\r\e[0K" ``` -Depending on the shell that your runner uses, for example if it is using ZSH, you may need to +Depending on the shell that your runner uses, for example if it is using Zsh, you may need to escape the special characters like so: `\\e` and `\\r`. In the example above: @@ -337,7 +337,7 @@ In the example above: - `my_first_section`: The name given to the section. - `\r\e[0K`: Prevents the section markers from displaying in the rendered (colored) job log, but they are displayed in the raw job log. To see them, in the top right - of the job log, click **{doc-text}** (**Show complete raw**). + of the job log, select **{doc-text}** (**Show complete raw**). - `\r`: carriage return. - `\e[0K`: clear line ANSI escape code. diff --git a/doc/ci/metrics_reports.md b/doc/ci/metrics_reports.md index 542d55665c7..03869a639f1 100644 --- a/doc/ci/metrics_reports.md +++ b/doc/ci/metrics_reports.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9788) in GitLab 11.10. Requires GitLab Runner 11.10 and above. -GitLab provides a lot of great reporting tools for things like [merge requests](../user/project/merge_requests/index.md) - [Unit test reports](unit_test_reports.md), [code quality](../user/project/merge_requests/code_quality.md), and performance tests. While JUnit is a great open framework for tests that "pass" or "fail", it is also important to see other types of metrics from a given change. +GitLab provides a lot of great reporting tools for things like [merge requests](../user/project/merge_requests/index.md) - [Unit test reports](testing/unit_test_reports.md), [code quality](../user/project/merge_requests/code_quality.md), and performance tests. While JUnit is a great open framework for tests that "pass" or "fail", it is also important to see other types of metrics from a given change. You can configure your job to use custom Metrics Reports, and GitLab displays a report on the merge request so that it's easier and faster to identify changes without having to check the entire log. diff --git a/doc/ci/pipeline_editor/index.md b/doc/ci/pipeline_editor/index.md index 57a9d7a9358..d87b336224c 100644 --- a/doc/ci/pipeline_editor/index.md +++ b/doc/ci/pipeline_editor/index.md @@ -53,12 +53,8 @@ reflected in the CI lint. It displays the same results as the existing [CI Lint ## View included CI/CD configuration -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7064) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `pipeline_editor_file_tree`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) -named `pipeline_editor_file_tree`. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7064) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `pipeline_editor_file_tree`. Disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357219) in GitLab 15.1. You can review configuration added with the [`include`](../yaml/index.md#include) keyword in the pipeline editor. In the top right, select the file tree (**{file-tree}**) diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index 2b18b1d353b..ff30d5701cc 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -74,6 +74,8 @@ If you set a quota for a subgroup, it is not used. ## View CI/CD minutes used by a group +> Displaying shared runners duration per project [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355666) in GitLab 15.0. + You can view the number of CI/CD minutes being used by a group. Prerequisite: @@ -88,6 +90,10 @@ To view CI/CD minutes being used for your group:  +The projects list shows projects with CI/CD minute usage or shared runners usage +in the current month only. The list includes all projects in the namespace and its +subgroups, sorted in descending order of CI/CD minute usage. + ## View CI/CD minutes used by a personal namespace > Displaying shared runners duration [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345795) in GitLab 15.0. @@ -98,6 +104,10 @@ You can view the number of CI/CD minutes being used by a personal namespace: 1. Select **Edit profile**. 1. On the left sidebar, select **Usage Quotas**. +The projects list shows [personal projects](../../user/project/working_with_projects.md#view-personal-projects) +with CI/CD minutes usage or shared runners usage in the current month only. The list +is sorted in descending order of CI/CD minute usage. + ## Purchase additional CI/CD minutes **(FREE SAAS)** If you're using GitLab SaaS, you can purchase additional packs of CI/CD minutes. @@ -129,6 +139,7 @@ so be sure to select the correct group. 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Settings > Usage Quotas**. +1. Select **Pipelines**. 1. Select **Buy additional minutes**. 1. Complete the details of the transaction. @@ -185,9 +196,9 @@ the duration of the pipeline if many jobs ran at the same time. The cost factor for a job running on a shared runner is: - `0.008` for public projects on GitLab SaaS, if [created 2021-07-17 or later](https://gitlab.com/gitlab-org/gitlab/-/issues/332708). - (For every 125 minutes of job time, you accrue 1 CD/CD minute.) + (For every 125 minutes of job time, you accrue 1 CI/CD minute.) - `0.008` for projects members of GitLab [Open Source program](../../subscriptions/index.md#gitlab-for-open-source). - (For every 125 minutes of job time, you accrue 1 CD/CD minute.) + (For every 125 minutes of job time, you accrue 1 CI/CD minute.) - `0` for public projects on GitLab self-managed instances, and for GitLab SaaS public projects created before 2021-07-17. - `1` for internal and private projects. @@ -198,7 +209,7 @@ GitLab SaaS shared runners have different cost factors, depending on the runner | GitLab SaaS runner type | Virtual machine configuration | CI/CD minutes cost factor | | :--------- | :------------------- | :--------- | | Linux OS + Docker executor| 1 vCPU, 3.75 GB RAM |1| -| macOS + shell executor | 4 vCPU, 10 GB RAM| 6 | +| macOS + shell executor | 4 vCPU, 10 GB RAM| 6 | ### Monthly reset of CI/CD minutes diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index c6142ebefc5..76419e61661 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -287,9 +287,15 @@ page, then selecting **Delete**.  +Deleting a pipeline does not automatically delete its +[child pipelines](parent_child_pipelines.md). +See the [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/39503) +for details. + WARNING: -Deleting a pipeline expires all pipeline caches, and deletes all related objects, -such as builds, logs, artifacts, and triggers. **This action cannot be undone.** +Deleting a pipeline expires all pipeline caches, and deletes all immediately +related objects, such as builds, logs, artifacts, and triggers. +**This action cannot be undone.** ### Pipeline security on protected branches @@ -451,12 +457,15 @@ For information on adding pipeline badges to projects, see [Pipeline badges](set ### Downstream pipelines -> Cancel or retry downstream pipelines from the graph view [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354974) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `downstream_retry_action`. Disabled by default. - In the pipeline graph view, downstream pipelines ([Multi-project pipelines](multi_project_pipelines.md) and [Parent-child pipelines](parent_child_pipelines.md)) display as a list of cards on the right of the graph. +#### Cancel or retry downstream pipelines from the graph view + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354974) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `downstream_retry_action`. Disabled by default. +> - [Generally available and feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357406) in GitLab 15.1. + To cancel a downstream pipeline that is still running, select **Cancel** (**{cancel}**) on the pipeline's card. diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index fa8041671dc..b0336d0499f 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -375,13 +375,14 @@ job artifacts are deleted. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/229936) in GitLab 13.4. > - [Made optional with a CI/CD setting](https://gitlab.com/gitlab-org/gitlab/-/issues/241026) in GitLab 13.8. -By default artifacts are always kept for the most recent successful pipeline for +By default artifacts are always kept for successful pipelines for the most recent commit on each ref. This means that the latest artifacts do not immediately expire according to the `expire_in` specification. -If a new pipeline for the same ref completes successfully, the previous pipeline's +If a pipeline for a new commit on the same ref completes successfully, the previous pipeline's artifacts are deleted according to the `expire_in` configuration. The artifacts -of the new pipeline are kept automatically. +of the new pipeline are kept automatically. If multiple pipelines run for the most +recent commit on the ref, all artifacts are kept. Keeping the latest artifacts can use a large amount of storage space in projects with a lot of jobs or large artifacts. If the latest artifacts are not needed in diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index d200dde143c..e0fe59d1ab8 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9186) in GitLab 12.0. > - [Squash and merge](../../user/project/merge_requests/squash_and_merge.md) support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13001) in GitLab 12.6. -For more information about why you might want to use merge trains, read [How merge trains keep your master green](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). +For more information about why you might want to use merge trains, read [How starting merge trains improve efficiency for DevOps](https://about.gitlab.com/blog/2020/01/30/all-aboard-merge-trains/). When [merged results pipelines](merged_results_pipelines.md) are enabled, the pipeline jobs run as if the changes from your source branch have already diff --git a/doc/ci/pipelines/parent_child_pipelines.md b/doc/ci/pipelines/parent_child_pipelines.md index 3206345d757..3fd739087ec 100644 --- a/doc/ci/pipelines/parent_child_pipelines.md +++ b/doc/ci/pipelines/parent_child_pipelines.md @@ -48,6 +48,9 @@ include the child pipeline configuration. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For an overview, see [Parent-Child Pipelines feature demo](https://youtu.be/n8KpBSqZNbk). +NOTE: +The artifact containing the generated YAML file must not be larger than 5MB. + ## Examples The simplest case is [triggering a child pipeline](../yaml/index.md#trigger) using a diff --git a/doc/ci/pipelines/pipeline_efficiency.md b/doc/ci/pipelines/pipeline_efficiency.md index 94cfeff3acc..991b3aef76c 100644 --- a/doc/ci/pipelines/pipeline_efficiency.md +++ b/doc/ci/pipelines/pipeline_efficiency.md @@ -74,7 +74,7 @@ hosted with a paid cloud service may be provisioned with: The [Pipeline success and duration charts](index.md#pipeline-success-and-duration-charts) give information about pipeline runtime and failed job counts. -Tests like [unit tests](../unit_test_reports.md), integration tests, end-to-end tests, +Tests like [unit tests](../testing/unit_test_reports.md), integration tests, end-to-end tests, [code quality](../../user/project/merge_requests/code_quality.md) tests, and others ensure that problems are automatically found by the CI/CD pipeline. There could be many pipeline stages involved causing long runtimes. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 2a95a4e1421..40daf154a5f 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -31,19 +31,43 @@ To change the visibility of your pipelines and related features: 1. Select or clear the **Public pipelines** checkbox. When it is selected, pipelines and related features are visible: - - For **public** projects, to everyone. - - For **internal** projects, to all logged-in users except [external users](../../user/permissions.md#external-users). - - For **private** projects, to all project members (Guest or higher). + - For [**Public**](../../user/public_access.md) projects, to everyone. + - For **Internal** projects, to all logged-in users except [external users](../../user/permissions.md#external-users). + - For **Private** projects, to all project members (Guest or higher). When it is cleared: - - For **public** projects, job logs, job artifacts, the pipeline security dashboard, + - For **Public** projects, job logs, job artifacts, the pipeline security dashboard, and the **CI/CD** menu items are visible only to project members (Reporter or higher). Other users, including guest users, can only view the status of pipelines and jobs, and only when viewing merge requests or commits. - - For **internal** projects, pipelines are visible to all logged in users except [external users](../../user/permissions.md#external-users). + - For **Internal** projects, pipelines are visible to all logged in users except [external users](../../user/permissions.md#external-users). Related features are visible only to project members (Reporter or higher). - - For **private** projects, pipelines and related features are visible to project members (Reporter or higher) only. + - For **Private** projects, pipelines and related features are visible to project members (Reporter or higher) only. + +### Change pipeline visibility for non-project members in public projects + +You can control the visibility of pipelines for non-project members in [public projects](../../user/public_access.md). + +This setting has no effect when: + +- Project visibility is set to [**Internal** or **Private**](../../user/public_access.md), + because non-project members cannot access internal or private projects. +- The [**Public pipelines**](#change-which-users-can-view-your-pipelines) setting is disabled. + +To change the pipeline visibility for non-project members: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. For **CI/CD**, choose: + - **Only project members**: Only project members can view pipelines. + - **Everyone With Access**: Non-project members can also view pipelines. +1. Select **Save changes**. + +The [CI/CD permissions table](../../user/permissions.md#gitlab-cicd-permissions) +lists the pipeline features non-project members can access when **Everyone With Access** +is selected. ## Auto-cancel redundant pipelines @@ -92,7 +116,7 @@ For more information, see [Deployment safety](../environments/deployment_safety. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211339) in GitLab 13.6. A deployment job can fail because a newer one has run. If you retry the failed deployment job, the -environment could be overwritten with older source code. If you click **Retry**, a modal warns you +environment could be overwritten with older source code. If you select **Retry**, a modal warns you about this and asks for confirmation. For more information, see [Deployment safety](../environments/deployment_safety.md). @@ -273,7 +297,7 @@ Use this regex for commonly used test tools. <!-- vale gitlab.Spelling = NO --> - Simplecov (Ruby). Example: `\(\d+.\d+\%\) covered`. -- pytest-cov (Python). Example: `^TOTAL.+?(\d+\%)$`. +- pytest-cov (Python). Example: `(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$`. - Scoverage (Scala). Example: `Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)`. - `phpunit --coverage-text --colors=never` (PHP). Example: `^\s*Lines:\s*\d+.\d+\%`. - gcovr (C/C++). Example: `^TOTAL.*\s+(\d+\%)$`. @@ -287,6 +311,7 @@ Use this regex for commonly used test tools. - .NET (OpenCover). Example: `(Visited Points).*\((.*)\)`. - .NET (`dotnet test` line coverage). Example: `Total\s*\|\s*(\d+(?:\.\d+)?)`. - tarpaulin (Rust). Example: `^\d+.\d+% coverage`. +- Pester (PowerShell). Example: `Covered (\d+\.\d+%)`. <!-- vale gitlab.Spelling = YES --> diff --git a/doc/ci/quick_start/index.md b/doc/ci/quick_start/index.md index e35a042a320..0369824c92e 100644 --- a/doc/ci/quick_start/index.md +++ b/doc/ci/quick_start/index.md @@ -79,7 +79,7 @@ To create a `.gitlab-ci.yml` file: 1. On the left sidebar, select **Project information > Details**. 1. Above the file list, select the branch you want to commit to, - click the plus icon, then select **New file**: + select the plus icon, then select **New file**:  @@ -115,7 +115,7 @@ To create a `.gitlab-ci.yml` file: [predefined variables](../variables/predefined_variables.md) that populate when the job runs. -1. Click **Commit changes**. +1. Select **Commit changes**. The pipeline starts when the commit is committed. @@ -172,11 +172,11 @@ To view your pipeline:  -- To view a visual representation of your pipeline, click the pipeline ID. +- To view a visual representation of your pipeline, select the pipeline ID.  -- To view details of a job, click the job name, for example, `deploy-prod`. +- To view details of a job, select the job name, for example, `deploy-prod`.  diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index 56e8b96d11f..3ebd38df8db 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -58,7 +58,7 @@ The process of configuring Review Apps is as follows: 1. [Install](https://docs.gitlab.com/runner/install/) and [configure](https://docs.gitlab.com/runner/commands/) a runner to do deployment. 1. Set up a job in `.gitlab-ci.yml` that uses the [predefined CI/CD variable](../variables/index.md) `${CI_COMMIT_REF_SLUG}` to create dynamic environments and restrict it to run only on branches. - Alternatively, you can get a YML template for this job by [enabling review apps](#enable-review-apps-button) for your project. + Alternatively, you can get a YAML template for this job by [enabling review apps](#enable-review-apps-button) for your project. 1. Optionally, set a job that [manually stops](../environments/index.md#stop-an-environment) the Review Apps. ### Enable Review Apps button diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index e7165339ea0..efd78fac2c6 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -85,9 +85,9 @@ To protect or unprotect a runner: 1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Find the runner you want to protect or unprotect. Make sure it's enabled. -1. Click the pencil button. +1. Select the pencil button. 1. Check the **Protected** option. -1. Click **Save changes**. +1. Select **Save changes**.  @@ -113,7 +113,7 @@ To reset the registration token: 1. Go to the project's **Settings > CI/CD**. 1. Expand the **General pipelines settings** section. -1. Find the **Runner token** form field and click the **Reveal value** button. +1. Find the **Runner token** form field and select **Reveal value**. 1. Delete the value and save the form. 1. After the page is refreshed, expand the **Runners settings** section and check the registration token - it should be changed. @@ -193,16 +193,16 @@ To make a runner pick untagged jobs: 1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. 1. Find the runner you want to pick untagged jobs and make sure it's enabled. -1. Click the pencil button. +1. Select the pencil button. 1. Check the **Run untagged jobs** option. -1. Click the **Save changes** button for the changes to take effect. +1. Select **Save changes** for the changes to take effect. NOTE: The runner tags list can not be empty when it's not allowed to pick untagged jobs. Below are some example scenarios of different variations. -### runner runs only tagged jobs +### Runner runs only tagged jobs The following examples illustrate the potential impact of the runner being set to run only tagged jobs. @@ -222,7 +222,7 @@ Example 3: 1. The runner is configured to run only tagged jobs and has the `docker` tag. 1. A job that has no tags defined is executed and stuck. -### runner is allowed to run untagged jobs +### Runner is allowed to run untagged jobs The following examples illustrate the potential impact of the runner being set to run tagged and untagged jobs. @@ -281,6 +281,17 @@ variables: - echo "Hello runner selector feature" ``` +## Runner statuses + +A runner can have one of the following statuses. + +| Status | Description | +|--------|-------------| +| **online** | The runner has contacted GitLab within the last 2 hours and is available to run jobs. | +| **offline** | The runner has not contacted GitLab in more than 2 hours and is not available to run jobs. Check the runner to see if you can bring it online. | +| **stale** | The runner has not contacted GitLab in more than 3 months. If the runner was created more than 3 months ago, but it never contacted the instance, it is also considered **stale**. | +| **never_contacted** | The runner has never contacted GitLab. To make the runner contact GitLab, run `gitlab-runner run`. | + ## Configure runner behavior with variables You can use [CI/CD variables](../variables/index.md) to configure runner Git behavior @@ -706,3 +717,175 @@ variables: | `ARTIFACT_COMPRESSION_LEVEL` | To adjust compression ratio, set to `fastest`, `fast`, `default`, `slow`, or `slowest`. This setting works with the Fastzip archiver only, so the GitLab Runner feature flag [`FF_USE_FASTZIP`](https://docs.gitlab.com/runner/configuration/feature-flags.html#available-feature-flags) must also be enabled. | | `CACHE_COMPRESSION_LEVEL` | To adjust compression ratio, set to `fastest`, `fast`, `default`, `slow`, or `slowest`. This setting works with the Fastzip archiver only, so the GitLab Runner feature flag [`FF_USE_FASTZIP`](https://docs.gitlab.com/runner/configuration/feature-flags.html#available-feature-flags) must also be enabled. | | `CACHE_REQUEST_TIMEOUT` | Configure the maximum duration of cache upload and download operations for a single job in minutes. Default is `10` minutes. | + +## Artifact attestation + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28940) in GitLab Runner 15.1. + +GitLab Runner can generate and produce attestation metadata for all build artifacts. To enable this feature, you must set the `RUNNER_GENERATE_ARTIFACTS_METADATA` environment variable to `true`. This variable can either be set globally or it can be set for individual jobs. The metadata is in rendered in a plain text `.json` file that's stored with the artifact. The file name is as follows: `{JOB_ID}-artifacts-metadata.json`. + +### Attestation format + +The attestation metadata is generated in the [in-toto attestation format](https://github.com/in-toto/attestation) for spec version [v0.1](https://in-toto.io/Statement/v0.1). The following fields are populated by default: + +| Field | Value | +| ------ | ------ | +| `_type` | `https://in-toto.io/Statement/v0.1` | +| `subject.name` | The filename of the artifact. | +| `subject.digest.sha256` | The artifact's `sha256` checksum. | +| `predicateType` | `https://slsa.dev/provenance/v0.2` | +| `predicate.buildType` | `https://gitlab.com/gitlab-org/gitlab-runner/-/blob/{GITLAB_RUNNER_VERSION}/PROVENANCE.md`. For example v15.0.0 | +| `predicate.builder.id` | A URI pointing to the runner details page, for example `https://gitlab.com/gitlab-com/www-gitlab-com/-/runners/3785264`. | +| `predicate.invocation.configSource.uri` | ``https://gitlab.example.com/.../{PROJECT_NAME}`` | +| `predicate.invocation.configSource.digest.sha256` | The repository's `sha256` checksum. | +| `predicate.invocation.configSource.entryPoint` | The name of the CI job that triggered the build. | +| `predicate.invocation.environment.name` | The name of the runner. | +| `predicate.invocation.environment.executor` | The runner executor. | +| `predicate.invocation.environment.architecture` | The architecture on which the CI job is run. | +| `predicate.invocation.parameters` | The names of any CI/CD or environment variables that were present when the build command was run. The value is always represented as an empty string to avoid leaking any secrets. | +| `metadata.buildStartedOn` | The time when the build was started. `RFC3339` formatted. | +| `metadata.buildEndedOn` | The time when the build ended. Since metadata generation happens during the build this moment in time will be slightly earlier than the one reported in GitLab. `RFC3339` formatted. | +| `metadata.reproducible` | Whether the build is reproducible by gathering all the generated metadata. Always `false`. | +| `metadata.completeness.parameters` | Whether the parameters are supplied. Always `true`. | +| `metadata.completeness.environment` | Whether the builder's environment is reported. Always `true`. | +| `metadata.completeness.materials` | Whether the build materials are reported. Always `false`. | + +An example of an attestation that the GitLab Runner might generate is as follows: + +```yaml +{ + "_type": "https://gitlab.com/gitlab-org/gitlab-runner/-/blob/v15.1.0/PROVENANCE.md", + "subject": [ + { + "name": "script.sh", + "digest": { + "sha256": "f5ae5ced234922eebe6461d32228ba8ab9c3d0c0f3983a3bef707e6e1a1ab52a" + } + } + ], + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicate": { + "buildType": "https://gitlab.com/gitlab-org/gitlab-runner/-/blob/v15.1.0/PROVENANCE.md", + "builder": { + "id": "https://gitlab.com/ggeorgiev_gitlab/playground/-/runners/14811533" + }, + "invocation": { + "configSource": { + "uri": "https://gitlab.com/ggeorgiev_gitlab/playground", + "digest": { + "sha256": "f0582e2c9a16b5cc2cde90e8be8f1b50fd67c631" + }, + "entryPoint": "whoami shell" + }, + "environment": { + "name": "local", + "executor": "shell", + "architecture": "amd64" + }, + "parameters": { + "CI_PIPELINE_ID": "", + "CI_PIPELINE_URL": "", + // All other CI variable names are listed here. Values are always represented as empty strings to avoid leaking secrets. + } + }, + "metadata": { + "buildStartedOn": "2022-06-17T00:47:27+03:00", + "buildFinishedOn": "2022-06-17T00:47:28+03:00", + "completeness": { + "parameters": true, + "environment": true, + "materials": false + }, + "reproducible": false + }, + "materials": [] + } +} +``` + +### Staging directory + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3403) in GitLab Runner 15.0. + +If you do not want to archive cache and artifacts in the system's default temporary directory, you can specify a different directory. + +You might need to change the directory if your system's default temporary path has constraints. +If you use a fast disk for the directory location, it can also improve performance. + +To change the directory, set `ARCHIVER_STAGING_DIR` as a variable in your CI job, or use a runner variable when you register the runner (`gitlab register --env ARCHIVER_STAGING_DIR=<dir>`). + +The directory you specify is used as the location for downloading artifacts prior to extraction. If the `fastzip` archiver is +used, this location is also used as scratch space when archiving. + +### Configure `fastzip` to improve performance + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3130) in GitLab Runner 15.0. + +To tune `fastzip`, ensure the [`FF_USE_FASTZIP`](https://docs.gitlab.com/runner/configuration/feature-flags.html#available-feature-flags) flag is enabled. +Then use any of the following environment variables. + +| Variable | Description | +|---------------------------------|--------------------------------------------------------| +| `FASTZIP_ARCHIVER_CONCURRENCY` | The number of files to be concurrently compressed. Default is the number of CPUs available. | +| `FASTZIP_ARCHIVER_BUFFER_SIZE` | The buffer size allocated per concurrency for each file. Data exceeding this number moves to scratch space. Default is 2 MiB. | +| `FASTZIP_EXTRACTOR_CONCURRENCY` | The number of files to be concurrency decompressed. Default is the number of CPUs available. | + +Files in a zip archive are appended sequentially. This makes concurrent compression challenging. `fastzip` works around +this limitation by compressing files concurrently to disk first, and then copying the result back to zip archive +sequentially. + +To avoid writing to disk and reading the contents back for smaller files, a small buffer per concurrency is used. This setting +can be controlled with `FASTZIP_ARCHIVER_BUFFER_SIZE`. The default size for this buffer is 2 MiB, therefore, a +concurrency of 16 will allocate 32 MiB. Data that exceeds the buffer size will be written to and read back from disk. +Therefore, using no buffer, `FASTZIP_ARCHIVER_BUFFER_SIZE: 0`, and only scratch space is a valid option. + +`FASTZIP_ARCHIVER_CONCURRENCY` controls how many files are compressed concurrency. As mentioned above, this setting +therefore can increase how much memory is being used, but also how much temporary data is written to the scratch space. +The default is the number of CPUs available, but given the memory ramifications, this may not always be the best +setting. + +`FASTZIP_EXTRACTOR_CONCURRENCY` controls how many files are decompressed at once. Files from a zip archive can natively +be read from concurrency, so no additional memory is allocated in additional to what the decompressor requires. This +defaults to the number of CPUs available. + +## Clean up stale runners **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363012) in GitLab 15.1. + +You can clean up group runners that have been inactive for more than three months. + +Group runners are those that were created at the group level. + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. Turn on the **Enable stale runner cleanup** toggle. + +### View stale runner cleanup logs + +You can check the [Sidekiq logs](../../administration/logs.md#sidekiq-logs) to see the cleanup result. In Kibana you can use the following query: + +```json +{ + "query": { + "match_phrase": { + "json.class.keyword": "Ci::Runners::StaleGroupRunnersPruneCronWorker" + } + } +} +``` + +Filter entries where stale runners were removed: + +```json +{ + "query": { + "range": { + "json.extra.ci_runners_stale_group_runners_prune_cron_worker.total_pruned": { + "gte": 1, + "lt": null + } + } + } +} +``` diff --git a/doc/ci/runners/index.md b/doc/ci/runners/index.md index 257dc02805b..729de4d99d3 100644 --- a/doc/ci/runners/index.md +++ b/doc/ci/runners/index.md @@ -7,16 +7,16 @@ type: reference # Runner SaaS **(FREE SAAS)** -If you are using self-managed GitLab or you use GitLab.com but want to use your own runners, you can -[install and configure your own runners](https://docs.gitlab.com/runner/install/). - -If you are using GitLab SaaS (GitLab.com), your CI jobs automatically run on runners provided by GitLab. +If you use GitLab SaaS (GitLab.com), your CI jobs automatically run on runners provided by GitLab. No configuration is required. Your jobs can run on: - [Linux runners](saas/linux_saas_runner.md). - [Windows runners](saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). -- [macOS runners](saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). +- [macOS runners](saas/macos_saas_runner.md) ([Limited Availability](../../policy/alpha-beta-support.md#limited-availability-la)). The number of minutes you can use on these runners depends on the [maximum number of CI/CD minutes](../pipelines/cicd_minutes.md) in your [subscription plan](https://about.gitlab.com/pricing/). + +If you use self-managed GitLab or you use GitLab.com but want to use your own runners, you can +[install and configure your own runners](https://docs.gitlab.com/runner/install/). diff --git a/doc/ci/runners/runners_scope.md b/doc/ci/runners/runners_scope.md index 6082a17d001..8b2753d26f1 100644 --- a/doc/ci/runners/runners_scope.md +++ b/doc/ci/runners/runners_scope.md @@ -84,7 +84,7 @@ To disable shared runners for a group: 1. Go to the group's **Settings > CI/CD** and expand the **Runners** section. 1. In the **Shared runners** area, turn off the **Enable shared runners for this group** toggle. 1. Optionally, to allow shared runners to be enabled for individual projects or subgroups, - click **Allow projects and subgroups to override the group setting**. + select **Allow projects and subgroups to override the group setting**. NOTE: To re-enable the shared runners for a group, turn on the @@ -200,11 +200,11 @@ You must have the Owner role for the group. 1. Go to the group you want to remove or pause the runner for. 1. On the left sidebar, select **CI/CD > Runners**. -1. Click **Pause** or **Remove runner**. +1. Select **Pause** or **Remove runner**. - If you pause a group runner that is used by multiple projects, the runner pauses for all projects. - From the group view, you cannot remove a runner that is assigned to more than one project. You must remove it from each project first. -1. On the confirmation dialog, click **OK**. +1. On the confirmation dialog, select **OK**. ## Specific runners @@ -226,33 +226,47 @@ A fork *does* copy the CI/CD settings of the cloned repository. ### Create a specific runner You can create a specific runner for your self-managed GitLab instance or for GitLab.com. -You must have the Owner role for the project. + +Prerequisite: + +- You must have at least the Maintainer role for the project. To create a specific runner: -1. [Install runner](https://docs.gitlab.com/runner/install/). -1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. -1. Note the URL and token. +1. [Install GitLab Runner](https://docs.gitlab.com/runner/install/). +1. On the top bar, select **Menu > Projects** and find the project where you want to use the runner. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. In the **Specific runners** section, note the URL and token. 1. [Register the runner](https://docs.gitlab.com/runner/register/). -### Enable a specific runner for a specific project +The runner is now enabled for the project. -A specific runner is available in the project it was created for. An administrator can -enable a specific runner to apply to additional projects. +### Enable a specific runner for a different project -- You must have the Owner role for the - project. +After a specific runner is created, you can enable it for other projects. + +Prerequisites: +You must have at least the Maintainer role for: + +- The project where the runner is already enabled. +- The project where you want to enable the runner. - The specific runner must not be [locked](#prevent-a-specific-runner-from-being-enabled-for-other-projects). -To enable or disable a specific runner for a project: +To enable a specific runner for a project: -1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. -1. Click **Enable for this project** or **Disable for this project**. +1. On the top bar, select **Menu > Projects** and find the project where you want to enable the runner. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **General pipelines**. +1. Expand **Runners**. +1. By the runner you want, select **Enable for this project**. You can edit a specific runner from any of the projects it's enabled for. -The modifications, which include unlocking, editing tags and the description, +The modifications, which include unlocking and editing tags and the description, affect all projects that use the runner. +An administrator can [enable the runner for multiple projects](../../user/admin_area/settings/continuous_integration.md#enable-a-specific-runner-for-multiple-projects). + ### Prevent a specific runner from being enabled for other projects You can configure a specific runner so it is "locked" and cannot be enabled for other projects. @@ -261,8 +275,9 @@ but can also be changed later. To lock or unlock a specific runner: -1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. +1. Go to the project's **Settings > CI/CD**. +1. Expand the **Runners** section. 1. Find the specific runner you want to lock or unlock. Make sure it's enabled. You cannot lock shared or group runners. -1. Click the pencil button. +1. Select **Edit** (**{pencil}**). 1. Check the **Lock to current projects** option. -1. Click **Save changes**. +1. Select **Save changes**. diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index fd3fedbc3f5..f1e28f7e74d 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -18,7 +18,7 @@ region of the VMs is US East1. Each instance is used only for one job. This ensures that any sensitive data left on the system can't be accessed by other people's CI/CD jobs. NOTE: -The final disk space your jobs can use will be less than 25GB. Some disk space allocated to the instance will be occupied by the operating system, the Docker image, and a copy of your cloned repository. +The final disk space your jobs can use will be less than 25GB. Some disk space allocated to the instance will be occupied by the operating system, the Docker image, and a copy of your cloned repository. The `gitlab-shared-runners-manager-X.gitlab.com` fleet of runners are dedicated for GitLab projects as well as community forks of them. They use a slightly larger machine type (n1-standard-2) and have a bigger SSD disk size. They don't run untagged jobs and unlike the general fleet of shared runners, the instances are re-used up to 40 times. @@ -55,7 +55,7 @@ To use this feature, define a [CI/CD variable](../../../ci/variables/index.md#cu `CI_PRE_CLONE_SCRIPT` that contains a bash script. NOTE: -The `CI_PRE_CLONE_SCRIPT` variable does not work on Windows runners. +The `CI_PRE_CLONE_SCRIPT` variable does not work on GitLab SaaS Windows or macOS Runners. ### Pre-clone script example diff --git a/doc/ci/runners/saas/macos/codesigning.md b/doc/ci/runners/saas/macos/codesigning.md index 4f8316faf17..71fdc61f0b6 100644 --- a/doc/ci/runners/saas/macos/codesigning.md +++ b/doc/ci/runners/saas/macos/codesigning.md @@ -6,13 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Code signing for SaaS runners on macOS -> Introduced in GitLab 15.0. - Before you can integrate GitLab with Apple services, install to a device, or deploy to the Apple App Store, you must [code sign](https://developer.apple.com/support/code-signing/) your application. To code sign an iOS project, you need the following files: -- A certifcate issued by Apple. +- A certificate issued by Apple. - A provisioning profile. ## Code signing iOS Projects with fastlane @@ -61,7 +59,7 @@ To use fastlane to code sign your application: get_provisioning_profile # match(type: "appstore",read_only: true) gym - upload_to_testflight + upload_to_testflight end end ``` diff --git a/doc/ci/runners/saas/macos_saas_runner.md b/doc/ci/runners/saas/macos_saas_runner.md index 0ab2de36f10..65404c89f9a 100644 --- a/doc/ci/runners/saas/macos_saas_runner.md +++ b/doc/ci/runners/saas/macos_saas_runner.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # SaaS runners on macOS (Limited Availability) **(PREMIUM SAAS)** -SaaS runners on macOS are now in [Limited Availability](../../../policy/alpha-beta-support.md#beta-features) for approved open source programs and customers in Premium and Ultimate plans. +SaaS runners on macOS are in [Limited Availability](../../../policy/alpha-beta-support.md#limited-availability-la) for approved open source programs and customers in Premium and Ultimate plans. SaaS runners on macOS provide an on-demand macOS build environment integrated with GitLab SaaS [CI/CD](../../../ci/index.md). @@ -18,6 +18,15 @@ CI/CD minutes used on GitLab SaaS macOS runners are included in your CI/CD minut Refer to the CI/CD minutes [cost factor](../../../ci/pipelines/cicd_minutes.md#cost-factor) for the cost factor applied to the GitLab SaaS macOS runners. +## Access request process + +While in limited availability, to run CI jobs on the macOS runners, GitLab SaaS customer namespaces must be explicitly added to the macOS `allow-list`. Customers who participated in the beta have already been added. + +After you have been added, you can use the macOS runners for any projects in your namespace. + +To request access, open a [limited availability access request](https://gitlab.com/gitlab-com/runner-saas-macos-limited-availability/-/issues/new). +The expected turnaround for activation is two business days. + ## Quickstart To start using SaaS runners on macOS, you must be an active GitLab SaaS Premium or Ultimate customer. Participants in the GitLab Open Source program are also eligible to use the service. diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index 2bf360e69f1..fb421ab8944 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -20,6 +20,8 @@ are stored securely outside of your project's repository, and are not version co It is safe to store sensitive information in these files. Secure files support both plain text and binary file types. +You can manage secure files in the project settings, or with the [secure files API](../../api/secure_files.md). + Secure files can be [downloaded and used by CI/CD jobs](#use-secure-files-in-cicd-jobs) by using the [load-secure-files](https://gitlab.com/gitlab-org/incubation-engineering/devops-for-mobile-apps/load-secure-files) tool. @@ -30,49 +32,14 @@ Additional features and capabilities are planned. ## Add a secure file to a project -To add a secure file to a project, use the [Secure Files API](../../api/secure_files.md#create-secure-file). - -Send a POST request to the secure files endpoint for your project: - -```shell -curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ - "https://gitlab.example.com/api/v4/projects/:project_id/secure_files" --form "name=myfile.jks" --form "file=@/path/to/file/myfile.jks" -``` - -The response returns all of the metadata for the file you just uploaded. For example: - -```json -{ - "id": 1, - "name": "myfile.jks", - "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", - "checksum_algorithm": "sha256", - "created_at": "2022-02-22T22:22:22.222Z" -} -``` - -## List all secure files in a project - -To list all secure files in a project, use the [Secure Files API](../../api/secure_files.md#list-project-secure-files). +To add a secure file to a project: -Send a GET request to the secure files endpoint for your project: - -```shell -curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" \ - "https://gitlab.example.com/api/v4/projects/:project_id/secure_files" -``` - -The response returns an array of all of the secure files in the project. For example: - -```json -[{ - "id": 1, - "name": "myfile.jks", - "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aac", - "checksum_algorithm": "sha256", - "created_at": "2022-02-22T22:22:22.222Z" -}] -``` +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. In the **Secure Files** section, select **Manage**. +1. Select **Upload File**. +1. Find the file to upload, select **Open**, and the file upload begins immediately. + The file shows up in the list when the upload is complete. ## Use secure files in CI/CD jobs diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index e876c6d7326..3ab814200fb 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -201,7 +201,7 @@ end-to-end-tests: alias: firefox - name: registry.gitlab.com/organization/private-api:latest alias: backend-api - - postgres:9.6.19 + - postgres:14.3 variables: FF_NETWORK_PER_BUILD: 1 POSTGRES_PASSWORD: supersecretpassword diff --git a/doc/ci/services/postgres.md b/doc/ci/services/postgres.md index 0bd43917cd1..c2ff4c60771 100644 --- a/doc/ci/services/postgres.md +++ b/doc/ci/services/postgres.md @@ -46,7 +46,7 @@ If you're wondering why we used `postgres` for the `Host`, read more at [How services are linked to the job](../services/index.md#how-services-are-linked-to-the-job). You can also use any other Docker image available on [Docker Hub](https://hub.docker.com/_/postgres). -For example, to use PostgreSQL 9.3, the service becomes `postgres:9.3`. +For example, to use PostgreSQL 14.3, the service becomes `postgres:14.3`. The `postgres` image can accept some environment variables. For more details, see the documentation on [Docker Hub](https://hub.docker.com/_/postgres). diff --git a/doc/ci/img/junit_test_report.png b/doc/ci/testing/img/junit_test_report.png Binary files differindex a4b98c8b910..a4b98c8b910 100644 --- a/doc/ci/img/junit_test_report.png +++ b/doc/ci/testing/img/junit_test_report.png diff --git a/doc/ci/img/pipelines_junit_test_report_v13_10.png b/doc/ci/testing/img/pipelines_junit_test_report_v13_10.png Binary files differindex ef79a2547af..ef79a2547af 100644 --- a/doc/ci/img/pipelines_junit_test_report_v13_10.png +++ b/doc/ci/testing/img/pipelines_junit_test_report_v13_10.png diff --git a/doc/ci/img/pipelines_junit_test_report_with_errors_v13_10.png b/doc/ci/testing/img/pipelines_junit_test_report_with_errors_v13_10.png Binary files differindex cfcf3bec76c..cfcf3bec76c 100644 --- a/doc/ci/img/pipelines_junit_test_report_with_errors_v13_10.png +++ b/doc/ci/testing/img/pipelines_junit_test_report_with_errors_v13_10.png diff --git a/doc/ci/testing/index.md b/doc/ci/testing/index.md new file mode 100644 index 00000000000..52af329873f --- /dev/null +++ b/doc/ci/testing/index.md @@ -0,0 +1,35 @@ +--- +stage: Verify +group: Pipeline Insights +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Test with GitLab CI/CD and generate reports in merge requests **(FREE)** + +Use GitLab CI/CD to test the changes included in a feature branch. You can also +display reports or link to important information directly from [merge requests](../../user/project/merge_requests/index.md). + +| Feature | Description | +|-------------------------------------------------------------------------------------------------|-------------| +| [Accessibility Testing](../../user/project/merge_requests/accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests. | +| [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md) | Quickly determine the browser performance impact of pending code changes. | +| [Load Performance Testing](../../user/project/merge_requests/load_performance_testing.md) | Quickly determine the server performance impact of pending code changes. | +| [Code Quality](../../user/project/merge_requests/code_quality.md) | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | +| [Display arbitrary job artifacts](../yaml/index.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../pipelines/job_artifacts.md) in merge requests. | +| [Unit test reports](unit_test_reports.md) | Configure your CI jobs to use Unit test reports, and let GitLab display a report on the merge request so that it's easier and faster to identify the failure without having to check the entire job log. | +| [License Compliance](../../user/compliance/license_compliance/index.md) | Manage the licenses of your dependencies. | +| [Metrics Reports](../metrics_reports.md) | Display the Metrics Report on the merge request so that it's fast and easier to identify changes to important metrics. | +| [Test Coverage visualization](../../user/project/merge_requests/test_coverage_visualization.md) | See test coverage results for merge requests, in the file diff. | +| [Fail fast testing](../../user/project/merge_requests/fail_fast_testing.md#fail-fast-testing) | Run a subset of your RSpec test suite, so failed tests stop the pipeline before the full suite of tests run, saving resources. | + +## Security Reports **(ULTIMATE)** + +In addition to the reports listed above, GitLab can do many types of [Security reports](../../user/application_security/index.md), +generated by scanning and reporting any vulnerabilities found in your project: + +| Feature | Description | +|----------------------------------------------------------------------------------------------|-------------| +| [Container Scanning](../../user/application_security/container_scanning/index.md) | Analyze your Docker images for known vulnerabilities. | +| [Dynamic Application Security Testing (DAST)](../../user/application_security/dast/index.md) | Analyze your running web applications for known vulnerabilities. | +| [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) | Analyze your dependencies for known vulnerabilities. | +| [Static Application Security Testing (SAST)](../../user/application_security/sast/index.md) | Analyze your source code for known vulnerabilities. | diff --git a/doc/ci/testing/unit_test_report_examples.md b/doc/ci/testing/unit_test_report_examples.md new file mode 100644 index 00000000000..a54deb254b7 --- /dev/null +++ b/doc/ci/testing/unit_test_report_examples.md @@ -0,0 +1,266 @@ +--- +stage: Verify +group: Pipeline Insights +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Unit test report examples **(FREE)** + +[Unit test reports](unit_test_reports.md) can be generated for many languages and packages. +Use these examples as guidelines for configuring your pipeline to generate unit test reports +for the listed languages and packages. You might need to edit the examples to match +the version of the language or package you are using. + +## Ruby + +Use the following job in `.gitlab-ci.yml`. This includes the `artifacts:paths` keyword to provide a link to the Unit test report output file. + +```yaml +## Use https://github.com/sj26/rspec_junit_formatter to generate a JUnit report format XML file with rspec +ruby: + stage: test + script: + - bundle install + - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml + artifacts: + when: always + paths: + - rspec.xml + reports: + junit: rspec.xml +``` + +## Go + +Use the following job in `.gitlab-ci.yml`: + +```yaml +## Use https://github.com/gotestyourself/gotestsum to generate a JUnit report format XML file with go +golang: + stage: test + script: + - go get gotest.tools/gotestsum + - gotestsum --junitfile report.xml --format testname + artifacts: + when: always + reports: + junit: report.xml +``` + +## Java + +There are a few tools that can produce JUnit report format XML file in Java. + +### Gradle + +In the following example, `gradle` is used to generate the test reports. +If there are multiple test tasks defined, `gradle` generates multiple +directories under `build/test-results/`. In that case, you can leverage glob +matching by defining the following path: `build/test-results/test/**/TEST-*.xml`: + +```yaml +java: + stage: test + script: + - gradle test + artifacts: + when: always + reports: + junit: build/test-results/test/**/TEST-*.xml +``` + +In [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620) +and later, you can use `**`. + +### Maven + +For parsing [Surefire](https://maven.apache.org/surefire/maven-surefire-plugin/) +and [Failsafe](https://maven.apache.org/surefire/maven-failsafe-plugin/) test +reports, use the following job in `.gitlab-ci.yml`: + +```yaml +java: + stage: test + script: + - mvn verify + artifacts: + when: always + reports: + junit: + - target/surefire-reports/TEST-*.xml + - target/failsafe-reports/TEST-*.xml +``` + +## Python example + +This example uses pytest with the `--junitxml=report.xml` flag to format the output +into the JUnit report XML format: + +```yaml +pytest: + stage: test + script: + - pytest --junitxml=report.xml + artifacts: + when: always + reports: + junit: report.xml +``` + +## C/C++ + +There are a few tools that can produce JUnit report format XML files in C/C++. + +### GoogleTest + +In the following example, `gtest` is used to generate the test reports. +If there are multiple `gtest` executables created for different architectures (`x86`, `x64` or `arm`), +you are required to run each test providing a unique filename. The results +are then aggregated together. + +```yaml +cpp: + stage: test + script: + - gtest.exe --gtest_output="xml:report.xml" + artifacts: + when: always + reports: + junit: report.xml +``` + +### CUnit + +[CUnit](https://cunity.gitlab.io/cunit/) can be made to produce [JUnit report format XML files](https://cunity.gitlab.io/cunit/group__CI.html) +automatically when run using its `CUnitCI.h` macros: + +```yaml +cunit: + stage: test + script: + - ./my-cunit-test + artifacts: + when: always + reports: + junit: ./my-cunit-test.xml +``` + +## .NET + +The [JunitXML.TestLogger](https://www.nuget.org/packages/JunitXml.TestLogger/) NuGet +package can generate test reports for .Net Framework and .Net Core applications. The following +example expects a solution in the root folder of the repository, with one or more +project files in sub-folders. One result file is produced per test project, and each file +is placed in the artifacts folder. This example includes optional formatting arguments, which +improve the readability of test data in the test widget. A full .Net Core +[example is available](https://gitlab.com/Siphonophora/dot-net-cicd-test-logging-demo). + +```yaml +## Source code and documentation are here: https://github.com/spekt/junit.testlogger/ + +Test: + stage: test + script: + - 'dotnet test --test-adapter-path:. --logger:"junit;LogFilePath=..\artifacts\{assembly}-test-result.xml;MethodFormat=Class;FailureBodyFormat=Verbose"' + artifacts: + when: always + paths: + - ./**/*test-result.xml + reports: + junit: + - ./**/*test-result.xml +``` + +## JavaScript + +There are a few tools that can produce JUnit report format XML files in JavaScript. + +### Jest + +The [jest-junit](https://github.com/jest-community/jest-junit) npm package can generate +test reports for JavaScript applications. In the following `.gitlab-ci.yml` example, +the `javascript` job uses Jest to generate the test reports: + +```yaml +javascript: + stage: test + script: + - 'jest --ci --reporters=default --reporters=jest-junit' + artifacts: + when: always + reports: + junit: + - junit.xml +``` + +### Karma + +The [Karma-junit-reporter](https://github.com/karma-runner/karma-junit-reporter) +npm package can generate test reports for JavaScript applications. In the following +`.gitlab-ci.yml` example, the `javascript` job uses Karma to generate the test reports: + +```yaml +javascript: + stage: test + script: + - karma start --reporters junit + artifacts: + when: always + reports: + junit: + - junit.xml +``` + +### Mocha + +The [JUnit Reporter for Mocha](https://github.com/michaelleeallen/mocha-junit-reporter) +NPM package can generate test reports for JavaScript applications. In the following +`.gitlab-ci.yml` example, the `javascript` job uses Mocha to generate the test reports: + +```yaml +javascript: + stage: test + script: + - mocha --reporter mocha-junit-reporter --reporter-options mochaFile=junit.xml + artifacts: + when: always + reports: + junit: + - junit.xml +``` + +## Flutter or Dart + +This example `.gitlab-ci.yml` file uses the [JUnit Report](https://pub.dev/packages/junitreport) +package to convert the `flutter test` output into JUnit report XML format: + +```yaml +test: + stage: test + script: + - flutter test --machine | tojunit -o report.xml + artifacts: + when: always + reports: + junit: + - report.xml +``` + +## PHP + +This example uses [PHPUnit](https://phpunit.de/) with the `--log-junit` flag. +You can also add this option using +[XML](https://phpunit.readthedocs.io/en/stable/configuration.html#the-junit-element) +in the `phpunit.xml` configuration file. + +```yaml +phpunit: + stage: test + script: + - composer install + - vendor/bin/phpunit --log-junit report.xml + artifacts: + when: always + reports: + junit: report.xml +``` diff --git a/doc/ci/testing/unit_test_reports.md b/doc/ci/testing/unit_test_reports.md new file mode 100644 index 00000000000..8aa41cd6fc0 --- /dev/null +++ b/doc/ci/testing/unit_test_reports.md @@ -0,0 +1,165 @@ +--- +stage: Verify +group: Pipeline Insights +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Unit test reports **(FREE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45318) in GitLab 11.2. Requires GitLab Runner 11.2 and above. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39737) from JUnit test reports to Unit test reports in GitLab 13.4. + +It is very common that a [CI/CD pipeline](../pipelines/index.md) contains a +test job that verifies your code. +If the tests fail, the pipeline fails and users get notified. The person that +works on the merge request has to check the job logs and see where the +tests failed so that they can fix them. + +You can configure your job to use Unit test reports, and GitLab displays a +report on the merge request so that it's easier and faster to identify the +failure without having to check the entire log. Unit test reports currently +only support test reports in the JUnit report format. + +If you don't use merge requests but still want to see the unit test report +output without searching through job logs, the full +[Unit test reports](#view-unit-test-reports-on-gitlab) are available +in the pipeline detail view. + +Consider the following workflow: + +1. Your default branch is rock solid, your project is using GitLab CI/CD and + your pipelines indicate that there isn't anything broken. +1. Someone from your team submits a merge request, a test fails and the pipeline + gets the known red icon. To investigate more, you have to go through the job + logs to figure out the cause of the failed test, which usually contain + thousands of lines. +1. You configure the Unit test reports and immediately GitLab collects and + exposes them in the merge request. No more searching in the job logs. +1. Your development and debugging workflow becomes easier, faster and efficient. + +## How it works + +First, GitLab Runner uploads all [JUnit report format XML files](https://www.ibm.com/docs/en/adfz/developer-for-zos/14.1.0?topic=formats-junit-xml-format) +as [artifacts](../yaml/artifacts_reports.md#artifactsreportsjunit) to GitLab. Then, when you visit a merge request, GitLab starts +comparing the head and base branch's JUnit report format XML files, where: + +- The base branch is the target branch (usually the default branch). +- The head branch is the source branch (the latest pipeline in each merge request). + +The reports panel has a summary showing how many tests failed, how many had errors +and how many were fixed. If no comparison can be done because data for the base branch +is not available, the panel just shows the list of failed tests for head. + +There are four types of results: + +1. **Newly failed tests:** Test cases which passed on base branch and failed on head branch +1. **Newly encountered errors:** Test cases which passed on base branch and failed due to a + test error on head branch +1. **Existing failures:** Test cases which failed on base branch and failed on head branch +1. **Resolved failures:** Test cases which failed on base branch and passed on head branch + +Each entry in the panel shows the test name and its type from the list +above. Clicking on the test name opens a modal window with details of its +execution time and the error output. + + + +### Number of recent failures + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241759) in merge requests in GitLab 13.7. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/268249) in GitLab 13.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235525) in Test Reports in GitLab 13.9. + +If a test failed in the project's default branch in the last 14 days, a message like +`Failed {n} time(s) in {default_branch} in the last 14 days` is displayed for that test. + +## How to set it up + +To enable the Unit test reports in merge requests, you must add +[`artifacts:reports:junit`](../yaml/artifacts_reports.md#artifactsreportsjunit) +in `.gitlab-ci.yml`, and specify the paths of the generated test reports. +The reports must be `.xml` files, otherwise [GitLab returns an Error 500](https://gitlab.com/gitlab-org/gitlab/-/issues/216575). + +In the following example for Ruby, the job in the `test` stage runs and GitLab +collects the unit test report from the job. After the job is executed, the +XML report is stored in GitLab as an artifact, and the results are shown in the +merge request widget. + +```yaml +## Use https://github.com/sj26/rspec_junit_formatter to generate a JUnit report format XML file with rspec +ruby: + stage: test + script: + - bundle install + - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml + artifacts: + when: always + paths: + - rspec.xml + reports: + junit: rspec.xml +``` + +To make the Unit test report output files browsable, include them with the +[`artifacts:paths`](../yaml/index.md#artifactspaths) keyword as well, as shown in the example. +To upload the report even if the job fails (for example if the tests do not pass), +use the [`artifacts:when:always`](../yaml/index.md#artifactswhen) keyword. + +You cannot have multiple tests with the same name and class in your JUnit report format XML file. + +In GitLab 15.0 and earlier, test reports from [parallel:matrix](../yaml/index.md#parallel:matrix) +jobs are aggregated together, which can cause some report information to not be displayed. +In GitLab 15.1 and later, [this bug is fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/296814), +and all report information is displayed. + +## View Unit test reports on GitLab + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5 behind a feature flag (`junit_pipeline_view`), disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/216478) in GitLab 13.3. + +If JUnit report format XML files are generated and uploaded as part of a pipeline, these reports +can be viewed inside the pipelines details page. The **Tests** tab on this page +displays a list of test suites and cases reported from the XML file. + + + +You can view all the known test suites and select each of these to see further +details, including the cases that make up the suite. + +You can also retrieve the reports via the [GitLab API](../../api/pipelines.md#get-a-pipelines-test-report). + +### Unit test reports parsing errors + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263457) in GitLab 13.10. + +If parsing JUnit report XML results in an error, an indicator is shown next to the job name. Hovering over the icon shows the parser error in a tooltip. If multiple parsing errors come from [grouped jobs](../jobs/index.md#group-jobs-in-a-pipeline), GitLab shows only the first error from the group. + + + +For test case parsing limits, see [Max test cases per unit test report](../../user/gitlab_com/#gitlab-cicd). + +GitLab does not parse very [large nodes](https://nokogiri.org/tutorials/parsing_an_html_xml_document.html#parse-options) of JUnit reports. There is [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/268035) open to make this optional. + +## View JUnit screenshots on GitLab + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0 behind the `:junit_pipeline_screenshots_view` feature flag, disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/216979) in GitLab 13.12. + +Upload your screenshots as [artifacts](../yaml/artifacts_reports.md#artifactsreportsjunit) to GitLab. If JUnit +report format XML files contain an `attachment` tag, GitLab parses the attachment. Note that: + +- The `attachment` tag **must** contain the relative path to `$CI_PROJECT_DIR` of the screenshots you uploaded. For + example: + + ```xml + <testcase time="1.00" name="Test"> + <system-out>[[ATTACHMENT|/path/to/some/file]]</system-out> + </testcase> + ``` + +- You should set the job that uploads the screenshot to + [`artifacts:when: always`](../yaml/index.md#artifactswhen) so that it still uploads a screenshot + when a test fails. + +A link to the test case attachment appears in the test case details in +[the pipeline test report](#view-unit-test-reports-on-gitlab). diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 1757543be5b..8d8afbffab9 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -26,8 +26,32 @@ experience (rather than the single file editor or the Web IDE). It includes: - The [CI/CD configuration visualization](pipeline_editor/index.md#visualize-ci-configuration), a graphical representation of your `.gitlab-ci.yml` file. -If you prefer to use another editor, you can use a schema like [the Schemastore `gitlab-ci` schema](https://json.schemastore.org/gitlab-ci) -with your editor of choice. +### Edit `.gitlab-ci.yml` locally + +If you prefer to edit your pipeline configuration locally, you can use the +GitLab CI/CD schema in your editor to verify basic syntax issues. Any +[editor with Schemastore support](https://www.schemastore.org/json/#editors) uses +the GitLab CI/CD schema by default. + +If you need to link to the schema directly, it +is at: + +```plaintext +https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json. +``` + +The schema rules for custom YAML tags like `!reference` will not work until the +custom tags are set in the editor settings. For example, in VS Code, you can set +`vscode-yaml` to parse `customTags`: + +```json +"yaml.customTags": [ + "!reference sequence" +] +``` + +To see the full list of custom tags covered by the CI/CD schema, check the +latest version of the schema, linked above. ### Verify syntax with CI Lint tool diff --git a/doc/ci/unit_test_reports.md b/doc/ci/unit_test_reports.md index 029bd20c553..7578a1e6009 100644 --- a/doc/ci/unit_test_reports.md +++ b/doc/ci/unit_test_reports.md @@ -1,395 +1,11 @@ --- -stage: Verify -group: Pipeline Insights -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'testing/unit_test_reports.md' +remove_date: '2022-08-31' --- -# Unit test reports **(FREE)** +This document was moved to [another location](testing/unit_test_reports.md). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45318) in GitLab 11.2. Requires GitLab Runner 11.2 and above. -> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39737) from JUnit test reports to Unit test reports in GitLab 13.4. - -It is very common that a [CI/CD pipeline](pipelines/index.md) contains a -test job that verifies your code. -If the tests fail, the pipeline fails and users get notified. The person that -works on the merge request has to check the job logs and see where the -tests failed so that they can fix them. - -You can configure your job to use Unit test reports, and GitLab displays a -report on the merge request so that it's easier and faster to identify the -failure without having to check the entire log. Unit test reports currently -only support test reports in the JUnit report format. - -If you don't use merge requests but still want to see the unit test report -output without searching through job logs, the full -[Unit test reports](#viewing-unit-test-reports-on-gitlab) are available -in the pipeline detail view. - -Consider the following workflow: - -1. Your default branch is rock solid, your project is using GitLab CI/CD and - your pipelines indicate that there isn't anything broken. -1. Someone from your team submits a merge request, a test fails and the pipeline - gets the known red icon. To investigate more, you have to go through the job - logs to figure out the cause of the failed test, which usually contain - thousands of lines. -1. You configure the Unit test reports and immediately GitLab collects and - exposes them in the merge request. No more searching in the job logs. -1. Your development and debugging workflow becomes easier, faster and efficient. - -## How it works - -First, GitLab Runner uploads all [JUnit report format XML files](https://www.ibm.com/docs/en/adfz/developer-for-zos/14.1.0?topic=formats-junit-xml-format) -as [artifacts](yaml/artifacts_reports.md#artifactsreportsjunit) to GitLab. Then, when you visit a merge request, GitLab starts -comparing the head and base branch's JUnit report format XML files, where: - -- The base branch is the target branch (usually the default branch). -- The head branch is the source branch (the latest pipeline in each merge request). - -The reports panel has a summary showing how many tests failed, how many had errors -and how many were fixed. If no comparison can be done because data for the base branch -is not available, the panel just shows the list of failed tests for head. - -There are four types of results: - -1. **Newly failed tests:** Test cases which passed on base branch and failed on head branch -1. **Newly encountered errors:** Test cases which passed on base branch and failed due to a - test error on head branch -1. **Existing failures:** Test cases which failed on base branch and failed on head branch -1. **Resolved failures:** Test cases which failed on base branch and passed on head branch - -Each entry in the panel shows the test name and its type from the list -above. Clicking on the test name opens a modal window with details of its -execution time and the error output. - - - -### Number of recent failures - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241759) in merge requests in GitLab 13.7. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/268249) in GitLab 13.8. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235525) in Test Reports in GitLab 13.9. - -If a test failed in the project's default branch in the last 14 days, a message like -`Failed {n} time(s) in {default_branch} in the last 14 days` is displayed for that test. - -## How to set it up - -To enable the Unit test reports in merge requests, you must add -[`artifacts:reports:junit`](yaml/artifacts_reports.md#artifactsreportsjunit) -in `.gitlab-ci.yml`, and specify the paths of the generated test reports. -The reports must be `.xml` files, otherwise [GitLab returns an Error 500](https://gitlab.com/gitlab-org/gitlab/-/issues/216575). - -In the following examples, the job in the `test` stage runs and GitLab -collects the Unit test report from each job. After each job is executed, the -XML reports are stored in GitLab as artifacts and their results are shown in the -merge request widget. - -To make the Unit test report output files browsable, include them with the -[`artifacts:paths`](yaml/index.md#artifactspaths) keyword as well, as shown in the [Ruby example](#ruby-example). -To upload the report even if the job fails (for example if the tests do not pass), use the [`artifacts:when:always`](yaml/index.md#artifactswhen) -keyword. - -You cannot have multiple tests with the same name and class in your JUnit report format XML file. - -### Ruby example - -Use the following job in `.gitlab-ci.yml`. This includes the `artifacts:paths` keyword to provide a link to the Unit test report output file. - -```yaml -## Use https://github.com/sj26/rspec_junit_formatter to generate a JUnit report format XML file with rspec -ruby: - stage: test - script: - - bundle install - - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml - artifacts: - when: always - paths: - - rspec.xml - reports: - junit: rspec.xml -``` - -### Go example - -Use the following job in `.gitlab-ci.yml`: - -```yaml -## Use https://github.com/gotestyourself/gotestsum to generate a JUnit report format XML file with go -golang: - stage: test - script: - - go get gotest.tools/gotestsum - - gotestsum --junitfile report.xml --format testname - artifacts: - when: always - reports: - junit: report.xml -``` - -### Java examples - -There are a few tools that can produce JUnit report format XML file in Java. - -#### Gradle - -In the following example, `gradle` is used to generate the test reports. -If there are multiple test tasks defined, `gradle` generates multiple -directories under `build/test-results/`. In that case, you can leverage glob -matching by defining the following path: `build/test-results/test/**/TEST-*.xml`: - -```yaml -java: - stage: test - script: - - gradle test - artifacts: - when: always - reports: - junit: build/test-results/test/**/TEST-*.xml -``` - -In [GitLab Runner 13.0](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2620) -and later, you can use `**`. - -#### Maven - -For parsing [Surefire](https://maven.apache.org/surefire/maven-surefire-plugin/) -and [Failsafe](https://maven.apache.org/surefire/maven-failsafe-plugin/) test -reports, use the following job in `.gitlab-ci.yml`: - -```yaml -java: - stage: test - script: - - mvn verify - artifacts: - when: always - reports: - junit: - - target/surefire-reports/TEST-*.xml - - target/failsafe-reports/TEST-*.xml -``` - -### Python example - -This example uses pytest with the `--junitxml=report.xml` flag to format the output -into the JUnit report XML format: - -```yaml -pytest: - stage: test - script: - - pytest --junitxml=report.xml - artifacts: - when: always - reports: - junit: report.xml -``` - -### C/C++ example - -There are a few tools that can produce JUnit report format XML files in C/C++. - -#### GoogleTest - -In the following example, `gtest` is used to generate the test reports. -If there are multiple `gtest` executables created for different architectures (`x86`, `x64` or `arm`), -you are required to run each test providing a unique filename. The results -are then aggregated together. - -```yaml -cpp: - stage: test - script: - - gtest.exe --gtest_output="xml:report.xml" - artifacts: - when: always - reports: - junit: report.xml -``` - -#### CUnit - -[CUnit](https://cunity.gitlab.io/cunit/) can be made to produce [JUnit report format XML files](https://cunity.gitlab.io/cunit/group__CI.html) automatically when run using its `CUnitCI.h` macros: - -```yaml -cunit: - stage: test - script: - - ./my-cunit-test - artifacts: - when: always - reports: - junit: ./my-cunit-test.xml -``` - -### .NET example - -The [JunitXML.TestLogger](https://www.nuget.org/packages/JunitXml.TestLogger/) NuGet -package can generate test reports for .Net Framework and .Net Core applications. The following -example expects a solution in the root folder of the repository, with one or more -project files in sub-folders. One result file is produced per test project, and each file -is placed in the artifacts folder. This example includes optional formatting arguments, which -improve the readability of test data in the test widget. A full .Net Core -[example is available](https://gitlab.com/Siphonophora/dot-net-cicd-test-logging-demo). - -```yaml -## Source code and documentation are here: https://github.com/spekt/junit.testlogger/ - -Test: - stage: test - script: - - 'dotnet test --test-adapter-path:. --logger:"junit;LogFilePath=..\artifacts\{assembly}-test-result.xml;MethodFormat=Class;FailureBodyFormat=Verbose"' - artifacts: - when: always - paths: - - ./**/*test-result.xml - reports: - junit: - - ./**/*test-result.xml -``` - -### JavaScript example - -There are a few tools that can produce JUnit report format XML files in JavaScript. - -#### Jest - -The [jest-junit](https://github.com/jest-community/jest-junit) npm package can generate test reports for JavaScript applications. -In the following `.gitlab-ci.yml` example, the `javascript` job uses Jest to generate the test reports: - -```yaml -javascript: - stage: test - script: - - 'jest --ci --reporters=default --reporters=jest-junit' - artifacts: - when: always - reports: - junit: - - junit.xml -``` - -#### Karma - -The [Karma-junit-reporter](https://github.com/karma-runner/karma-junit-reporter) npm package can generate test reports for JavaScript applications. -In the following `.gitlab-ci.yml` example, the `javascript` job uses Karma to generate the test reports: - -```yaml -javascript: - stage: test - script: - - karma start --reporters junit - artifacts: - when: always - reports: - junit: - - junit.xml -``` - -#### Mocha - -The [JUnit Reporter for Mocha](https://github.com/michaelleeallen/mocha-junit-reporter) NPM package can generate test reports for JavaScript -applications. -In the following `.gitlab-ci.yml` example, the `javascript` job uses Mocha to generate the test reports: - -```yaml -javascript: - stage: test - script: - - mocha --reporter mocha-junit-reporter --reporter-options mochaFile=junit.xml - artifacts: - when: always - reports: - junit: - - junit.xml -``` - -### Flutter / Dart example - -This example `.gitlab-ci.yml` file uses the [JUnit Report](https://pub.dev/packages/junitreport) package to convert the `flutter test` output into JUnit report XML format: - -```yaml -test: - stage: test - script: - - flutter test --machine | tojunit -o report.xml - artifacts: - when: always - reports: - junit: - - report.xml -``` - -### PHP example - -This example uses [PHPUnit](https://phpunit.de/) with the `--log-junit` flag. -You can also add this option using -[XML](https://phpunit.readthedocs.io/en/stable/configuration.html#the-junit-element) -in the `phpunit.xml` configuration file. - -```yaml -phpunit: - stage: test - script: - - composer install - - vendor/bin/phpunit --log-junit report.xml - artifacts: - when: always - reports: - junit: report.xml -``` - -## Viewing Unit test reports on GitLab - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24792) in GitLab 12.5 behind a feature flag (`junit_pipeline_view`), disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/216478) in GitLab 13.3. - -If JUnit report format XML files are generated and uploaded as part of a pipeline, these reports -can be viewed inside the pipelines details page. The **Tests** tab on this page -displays a list of test suites and cases reported from the XML file. - - - -You can view all the known test suites and select each of these to see further -details, including the cases that make up the suite. - -You can also retrieve the reports via the [GitLab API](../api/pipelines.md#get-a-pipelines-test-report). - -### Unit test reports parsing errors - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263457) in GitLab 13.10. - -If parsing JUnit report XML results in an error, an indicator is shown next to the job name. Hovering over the icon shows the parser error in a tooltip. If multiple parsing errors come from [grouped jobs](jobs/index.md#group-jobs-in-a-pipeline), GitLab shows only the first error from the group. - - - -For test case parsing limits, see [Max test cases per unit test report](../user/gitlab_com/#gitlab-cicd). - -GitLab does not parse very [large nodes](https://nokogiri.org/tutorials/parsing_an_html_xml_document.html#parse-options) of JUnit reports. There is [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/268035) open to make this optional. - -## Viewing JUnit screenshots on GitLab - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202114) in GitLab 13.0 behind the `:junit_pipeline_screenshots_view` feature flag, disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/216979) in GitLab 13.12. - -Upload your screenshots as [artifacts](yaml/artifacts_reports.md#artifactsreportsjunit) to GitLab. If JUnit -report format XML files contain an `attachment` tag, GitLab parses the attachment. Note that: - -- The `attachment` tag **must** contain the relative path to `$CI_PROJECT_DIR` of the screenshots you uploaded. For - example: - - ```xml - <testcase time="1.00" name="Test"> - <system-out>[[ATTACHMENT|/path/to/some/file]]</system-out> - </testcase> - ``` - -- You should set the job that uploads the screenshot to - [`artifacts:when: always`](yaml/index.md#artifactswhen) so that it still uploads a screenshot - when a test fails. - -A link to the test case attachment appears in the test case details in -[the pipeline test report](#viewing-unit-test-reports-on-gitlab). +<!-- This redirect file can be deleted after <2022-08-31>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index 90403351b11..c53fad69376 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -373,7 +373,7 @@ You can configure a project, group, or instance CI/CD variable to be available only to pipelines that run on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). -[Merged results pipelines](../pipelines/merge_request_pipelines.md#types-of-merge-request-pipelines), which run on a +[Merged results pipelines](../pipelines/merged_results_pipelines.md), which run on a temporary merge commit, not a branch or tag, do not have access to these variables. Pipelines that run directly on the merge request's source branch, with no added merge commit, can access @@ -526,6 +526,7 @@ export CI_PROJECT_ID="34" export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-foss" export CI_PROJECT_NAME="gitlab-foss" export CI_PROJECT_TITLE="GitLab FOSS" +export CI_PROJECT_DESCRIPTION="GitLab FOSS is a read-only mirror of GitLab, with all proprietary code removed." export CI_PROJECT_NAMESPACE="gitlab-org" export CI_PROJECT_ROOT_NAMESPACE="gitlab-org" export CI_PROJECT_PATH="gitlab-org/gitlab-foss" @@ -697,6 +698,10 @@ You can override the value of a variable when you: The pipeline variables declared in these events take [priority over other variables](#cicd-variable-precedence). +NOTE: +You should avoid overriding [predefined variables](predefined_variables.md), +as it can cause the pipeline to behave unexpectedly. + ### Override a variable when running a pipeline manually You can override the value of a CI/CD variable when you @@ -851,14 +856,16 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_SERVER_PROTOCOL=https ++ export CI_SERVER_NAME=GitLab ++ CI_SERVER_NAME=GitLab -++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal -++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_project_service_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ export GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal +++ GITLAB_FEATURES=audit_events,burndown_charts,code_owners,contribution_analytics,description_diffs,elastic_search,group_bulk_edit,group_burndown_charts,group_webhooks,issuable_default_templates,issue_weights,jenkins_integration,ldap_group_sync,member_lock,merge_request_approvers,multiple_issue_assignees,multiple_ldap_servers,multiple_merge_request_assignees,protected_refs_for_users,push_rules,related_issues,repository_mirrors,repository_size_limit,scoped_issue_board,usage_quotas,visual_review_app,wip_limits,adjourned_deletion_for_projects_and_groups,admin_audit_log,auditor_user,batch_comments,blocking_merge_requests,board_assignee_lists,board_milestone_lists,ci_cd_projects,cluster_deployments,code_analytics,code_owner_approval_required,commit_committer_check,cross_project_pipelines,custom_file_templates,custom_file_templates_for_namespace,custom_project_templates,custom_prometheus_metrics,cycle_analytics_for_groups,db_load_balancing,default_project_deletion_protection,dependency_proxy,deploy_board,design_management,email_additional_text,extended_audit_events,external_authorization_service_api_management,feature_flags,file_locks,geo,github_integration,group_allowed_email_domains,group_project_templates,group_saml,issues_analytics,jira_dev_panel_integration,ldap_group_sync_filter,merge_pipelines,merge_request_performance_metrics,merge_trains,metrics_reports,multiple_approval_rules,multiple_group_issue_boards,object_storage,operations_dashboard,packages,productivity_analytics,project_aliases,protected_environments,reject_unsigned_commits,required_ci_templates,scoped_labels,service_desk,smartcard_auth,group_timelogs,type_of_work_analytics,unprotection_restrictions,ci_project_subscriptions,cluster_health,container_scanning,dast,dependency_scanning,epics,group_ip_restriction,incident_management,insights,license_management,personal_access_token_expiration_policy,pod_logs,prometheus_alerts,report_approver_rules,sast,security_dashboard,tracing,web_ide_terminal ++ export CI_PROJECT_ID=17893 ++ CI_PROJECT_ID=17893 ++ export CI_PROJECT_NAME=ci-debug-trace ++ CI_PROJECT_NAME=ci-debug-trace ++ export CI_PROJECT_TITLE='GitLab FOSS' ++ CI_PROJECT_TITLE='GitLab FOSS' +++ export CI_PROJECT_DESCRIPTION='GitLab FOSS is a read-only mirror of GitLab, with all proprietary code removed.' +++ CI_PROJECT_DESCRIPTION='GitLab FOSS is a read-only mirror of GitLab, with all proprietary code removed.' ++ export CI_PROJECT_PATH=gitlab-examples/ci-debug-trace ++ CI_PROJECT_PATH=gitlab-examples/ci-debug-trace ++ export CI_PROJECT_PATH_SLUG=gitlab-examples-ci-debug-trace diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index f8f775a6df4..e6d61ef342b 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -16,6 +16,10 @@ with a `script` command. There are also a number of [variables you can use to configure runner behavior](../runners/configure_runners.md#configure-runner-behavior-with-variables) globally or for individual jobs. +NOTE: +You should avoid [overriding](index.md#override-a-defined-cicd-variable) predefined variables, +as it can cause the pipeline to behave unexpectedly. + | Variable | GitLab | Runner | Description | |------------------------------------------|--------|--------|-------------| | `CHAT_CHANNEL` | 10.6 | all | The Source chat channel that triggered the [ChatOps](../chatops/index.md) command. | @@ -92,6 +96,7 @@ There are also a number of [variables you can use to configure runner behavior]( | `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | A comma-separated, lowercase list of the languages used in the repository. For example `ruby,javascript,html,css`. | | `CI_PROJECT_ROOT_NAMESPACE` | 13.2 | 0.5 | The root project namespace (username or group name) of the job. For example, if `CI_PROJECT_NAMESPACE` is `root-group/child-group/grandchild-group`, `CI_PROJECT_ROOT_NAMESPACE` is `root-group`. | | `CI_PROJECT_TITLE` | 12.4 | all | The human-readable project name as displayed in the GitLab web interface. | +| `CI_PROJECT_DESCRIPTION` | 15.1 | all | The project description as displayed in the GitLab web interface. | | `CI_PROJECT_URL` | 8.10 | 0.5 | The HTTP(S) address of the project. | | `CI_PROJECT_VISIBILITY` | 10.3 | all | The project visibility. Can be `internal`, `private`, or `public`. | | `CI_PROJECT_CLASSIFICATION_LABEL` | 14.2 | all | The project [external authorization classification label](../../user/admin_area/settings/external_authorization.md). | @@ -112,7 +117,6 @@ There are also a number of [variables you can use to configure runner behavior]( | `CI_SERVER_PORT` | 12.8 | all | The port of the GitLab instance URL, without host or protocol. For example `8080`. | | `CI_SERVER_PROTOCOL` | 12.8 | all | The protocol of the GitLab instance URL, without host or port. For example `https`. | | `CI_SERVER_REVISION` | all | all | GitLab revision that schedules jobs. | -| `CI_SERVER_TLS_CA_FILE` | all | all | File containing the CA certificate to verify the GitLab server. | | `CI_SERVER_URL` | 12.7 | all | The base URL of the GitLab instance, including protocol and port. For example `https://gitlab.example.com:8080`. | | `CI_SERVER_VERSION_MAJOR` | 11.4 | all | The major version of the GitLab instance. For example, if the GitLab version is `13.6.1`, the `CI_SERVER_VERSION_MAJOR` is `13`. | | `CI_SERVER_VERSION_MINOR` | 11.4 | all | The minor version of the GitLab instance. For example, if the GitLab version is `13.6.1`, the `CI_SERVER_VERSION_MINOR` is `6`. | diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 64d61ec9f11..7a1ef61f109 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -7,7 +7,7 @@ type: reference # Where variables can be used **(FREE)** -As it's described in the [CI/CD variables](index.md) docs, you can +As it's described in the [CI/CD variables](index.md) documentation, you can define many different variables. Some of them can be used for all GitLab CI/CD features, but some of them are more or less limited. @@ -17,30 +17,30 @@ This document describes where and how the different types of variables can be us There are two places defined variables can be used. On the: -1. GitLab side, in `.gitlab-ci.yml`. +1. GitLab side, in the [`.gitlab-ci.yml` file](../yaml/index.md). 1. The GitLab Runner side, in `config.toml`. ### `.gitlab-ci.yml` file -| Definition | Can be expanded? | Expansion place | Description | -|:----------------------|:-----------------|:-----------------------|:------------| -| `after_script` | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | -| `artifacts:name` | yes | Runner | The variable expansion is made by GitLab Runner's shell environment. | -| `before_script` | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment) | -| `cache:key` | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | -| `environment:name` | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | -| `environment:url` | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.<br/><br/>Supported are all variables defined for a job (project/group variables, variables from `.gitlab-ci.yml`, variables from triggers, variables from pipeline schedules).<br/><br/>Not supported are variables defined in the GitLab Runner `config.toml` and variables created in the job's `script`. | -| `except:variables:[]` | no | n/a | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | -| `image` | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | -| `include` | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. <br/><br/>Predefined project variables are supported: `GITLAB_FEATURES`, `CI_DEFAULT_BRANCH`, and all variables that start with `CI_PROJECT_` (for example `CI_PROJECT_NAME`). | -| `only:variables:[]` | no | n/a | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | -| `resource_group` | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/>- `CI_ENVIRONMENT_URL`<br/>- [Persisted variables](#persisted-variables). | -| `rules:if` | no | n/a | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | -| `script` | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | -| `services:[]:name` | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | -| `services:[]` | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | -| `tags` | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35742) in GitLab 14.1. | -| `variables` | yes | GitLab/Runner | The variable expansion is first made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab, and then any unrecognized or unavailable variables are expanded by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | +| Definition | Can be expanded? | Expansion place | Description | +|:----------------------------------------------------------------------|:-----------------|:-----------------------|:------------| +| [`after_script`](../yaml/index.md#after_script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | +| [`artifacts:name`](../yaml/index.md#artifactsname) | yes | Runner | The variable expansion is made by GitLab Runner's shell environment. | +| [`before_script`](../yaml/index.md#before_script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment) | +| [`cache:key`](../yaml/index.md#cachekey) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | +| [`environment:name`](../yaml/index.md#environmentname) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`environment:url`](../yaml/index.md#environmenturl) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.<br/><br/>Supported are all variables defined for a job (project/group variables, variables from `.gitlab-ci.yml`, variables from triggers, variables from pipeline schedules).<br/><br/>Not supported are variables defined in the GitLab Runner `config.toml` and variables created in the job's `script`. | +| [`except:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`image`](../yaml/index.md#image) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | +| [`include`](../yaml/index.md#include) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. <br/><br/>Predefined project variables are supported: `GITLAB_FEATURES`, `CI_DEFAULT_BRANCH`, and all variables that start with `CI_PROJECT_` (for example `CI_PROJECT_NAME`). | +| [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`resource_group`](../yaml/index.md#resource_group) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:<br/>- `CI_ENVIRONMENT_URL`<br/>- [Persisted variables](#persisted-variables). | +| [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:<br/><br/>- Variables that are based on the environment's name (`CI_ENVIRONMENT_NAME`, `CI_ENVIRONMENT_SLUG`).<br/>- Any other variables related to environment (currently only `CI_ENVIRONMENT_URL`).<br/>- [Persisted variables](#persisted-variables). | +| [`script`](../yaml/index.md#script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | +| [`services:name`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | +| [`services`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | +| [`tags`](../yaml/index.md#tags) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35742) in GitLab 14.1. | +| [`variables`](../yaml/index.md#variables) | yes | GitLab/Runner | The variable expansion is first made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab, and then any unrecognized or unavailable variables are expanded by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | ### `config.toml` file diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index 289e8b91104..e4324ab06e1 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -85,10 +85,10 @@ GitLab can display the results of one or more reports in: ## `artifacts:reports:cobertura` (removed) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3708) in GitLab 12.9. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.9. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.7. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/348980) in GitLab 15.0. -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.9 and +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78132) in GitLab 14.7 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/348980) in GitLab 15.0. Use `artifacts:reports:coverage_report` instead. @@ -179,7 +179,7 @@ The collected Dependency Scanning report uploads to GitLab as an artifact. GitLab can display the results of one or more reports in: -- The merge request [dependency scanning widget](../../user/application_security/dependency_scanning/index.md#overview). +- The merge request [dependency scanning widget](../../user/application_security/dependency_scanning/index.md). - The pipeline [**Security** tab](../../user/application_security/security_dashboard/index.md#view-vulnerabilities-in-a-pipeline). - The [security dashboard](../../user/application_security/security_dashboard/index.md). - The [Project Vulnerability report](../../user/application_security/vulnerability_report/index.md). @@ -209,6 +209,7 @@ The exceptions to the [original dotenv rules](https://github.com/motdotla/dotenv self-managed instances is 150, and can be changed by changing the `dotenv_variables` [application limit](../../administration/instance_limits.md#limit-dotenv-variables). - Variable substitution in the `.env` file is not supported. +- [Multiline values in the `.env` file](https://github.com/motdotla/dotenv#multiline-values) are not supported. - The `.env` file can't have empty lines or comments (starting with `#`). - Key values in the `env` file cannot have spaces or newline characters (`\n`), including when using single or double quotes. - Quote escaping during parsing (`key = 'value'` -> `{key: "value"}`) is not supported. @@ -219,7 +220,7 @@ The `junit` report collects [JUnit report format XML files](https://www.ibm.com/ The collected Unit test reports upload to GitLab as an artifact. Although JUnit was originally developed in Java, there are many third-party ports for other languages such as JavaScript, Python, and Ruby. -See [Unit test reports](../unit_test_reports.md) for more details and examples. +See [Unit test reports](../testing/unit_test_reports.md) for more details and examples. Below is an example of collecting a JUnit report format XML file from Ruby's RSpec test tool: ```yaml @@ -235,8 +236,8 @@ rspec: GitLab can display the results of one or more reports in: -- The merge request [code quality widget](../../ci/unit_test_reports.md#how-it-works). -- The [full report](../../ci/unit_test_reports.md#viewing-unit-test-reports-on-gitlab). +- The merge request [code quality widget](../testing/unit_test_reports.md#how-it-works). +- The [full report](../testing/unit_test_reports.md#view-unit-test-reports-on-gitlab). Some JUnit tools export to multiple XML files. You can specify multiple test report paths in a single job to concatenate them into a single file. Use either: diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index eb587db6d5f..912eca364c9 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -175,6 +175,8 @@ Use `include:local` instead of symbolic links. - The YAML file must have the extension `.yml` or `.yaml`. - You can [use `*` and `**` wildcards in the file path](includes.md#use-includelocal-with-wildcard-file-paths). +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `include:local`**: ```yaml @@ -209,6 +211,8 @@ use `include:file`. You can use `include:file` in combination with `include:proj - A full path, relative to the root directory (`/`). The YAML file must have the extension `.yml` or `.yaml`. +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `include:file`**: ```yaml @@ -226,7 +230,7 @@ include: file: '/templates/.gitlab-ci-template.yml' - project: 'my-group/my-project' - ref: v1.0.0 + ref: v1.0.0 # Git Tag file: '/templates/.gitlab-ci-template.yml' - project: 'my-group/my-project' @@ -267,6 +271,8 @@ Use `include:remote` with a full URL to include a file from a different location - A public URL accessible by an HTTP/HTTPS `GET` request. Authentication with the remote URL is not supported. The YAML file must have the extension `.yml` or `.yaml`. +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `include:remote`**: ```yaml @@ -292,6 +298,8 @@ Use `include:template` to include [`.gitlab-ci.yml` templates](https://gitlab.co - [`.gitlab-ci.yml` templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `include:template`**: ```yaml @@ -331,6 +339,11 @@ The order of the items in `stages` defines the execution order for jobs: - Jobs in the same stage run in parallel. - Jobs in the next stage run after the jobs from the previous stage complete successfully. +If a pipeline contains only jobs in the `.pre` or `.post` stages, it does not run. +There must be at least one other job in a different stage. `.pre` and `.post` stages +can be used in [required pipeline configuration](../../user/admin_area/settings/continuous_integration.md#required-pipeline-configuration) +to define compliance jobs that must run before or after project pipeline jobs. + **Keyword type**: Global keyword. **Example of `stages`**: @@ -504,6 +517,8 @@ Use `after_script` to define an array of commands that run after each job, inclu - Long commands [split over multiple lines](script.md#split-long-commands). - [YAML anchors](yaml_optimization.md#yaml-anchors-for-scripts). +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `after_script`**: ```yaml @@ -809,8 +824,8 @@ If not defined, the default name is `artifacts`, which becomes `artifacts.zip` w **Possible inputs**: -- The name of the artifacts archive. Can use [CI/CD variables](../variables/index.md). Must be combined with - [`artifacts:paths`](#artifactspaths). +- The name of the artifacts archive. CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + Must be combined with [`artifacts:paths`](#artifactspaths). **Example of `artifacts:name`**: @@ -977,7 +992,7 @@ failure. - `on_success` (default): Upload artifacts only when the job succeeds. - `on_failure`: Upload artifacts only when the job fails. - `always`: Always upload artifacts (except when jobs time out). For example, when - [uploading artifacts](../unit_test_reports.md#viewing-junit-screenshots-on-gitlab) + [uploading artifacts](../testing/unit_test_reports.md#view-junit-screenshots-on-gitlab) required to troubleshoot failing tests. **Example of `artifacts:when`**: @@ -1002,6 +1017,8 @@ Use `before_script` to define an array of commands that should run before each j - Long commands [split over multiple lines](script.md#split-long-commands). - [YAML anchors](yaml_optimization.md#yaml-anchors-for-scripts). +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `before_script`**: ```yaml @@ -1016,6 +1033,7 @@ job: - Scripts you specify in `before_script` are concatenated with any scripts you specify in the main [`script`](#script). The combined scripts execute together in a single shell. +- Using `before_script` at the top level, but not in the `default` section, [is deprecated](#globally-defined-image-services-cache-before_script-after_script). **Related topics**: @@ -1087,7 +1105,7 @@ no `cache:key` share the `default` cache. **Possible inputs**: - A string. -- A [predefined variables](../variables/index.md). +- A predefined [CI/CD variable](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). - A combination of both. **Example of `cache:key`**: @@ -1504,7 +1522,8 @@ Common environment names are `qa`, `staging`, and `production`, but you can use formats: - Plain text, including letters, digits, spaces, and these characters: `-`, `_`, `/`, `$`, `{`, `}`. -- CI/CD variables, including predefined, project, group, instance, or variables defined in the +- [CI/CD variables](../variables/where_variables_can_be_used.md#gitlab-ciyml-file), + including predefined, project, group, instance, or variables defined in the `.gitlab-ci.yml` file. You can't use variables defined in a `script` section. **Example of `environment:name`**: @@ -1526,7 +1545,8 @@ Set a URL for an [environment](../environments/index.md). **Possible inputs**: A single URL, in one of these formats: - Plain text, like `https://prod.example.com`. -- CI/CD variables, including predefined, project, group, instance, or variables defined in the +- [CI/CD variables](../variables/where_variables_can_be_used.md#gitlab-ciyml-file), + including predefined, project, group, instance, or variables defined in the `.gitlab-ci.yml` file. You can't use variables defined in a `script` section. **Example of `environment:url`**: @@ -1788,6 +1808,8 @@ Use `image` to specify a Docker image that the job runs in. - `<image-name>:<tag>` - `<image-name>@<digest>` +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `image`**: ```yaml @@ -1861,6 +1883,52 @@ image: - [Override the entrypoint of an image](../docker/using_docker_images.md#override-the-entrypoint-of-an-image). +#### `image:pull_policy` + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21619) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. Disabled by default. +> - Requires GitLab Runner 15.1 or later. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_docker_image_pull_policy`. +The feature is not ready for production use. + +The pull policy that the runner uses to fetch the Docker image. + +**Keyword type**: Job keyword. You can use it only as part of a job or in the [`default` section](#default). + +**Possible inputs**: + +- A single pull policy, or multiple pull policies in an array. + Can be `always`, `if-not-present`, or `never`. + +**Examples of `image:pull_policy`**: + +```yaml +job1: + script: echo "A single pull policy." + image: + name: ruby:3.0 + pull_policy: if-not-present + +job2: + script: echo "Multiple pull policies." + image: + name: ruby:3.0 + pull_policy: [always, if-not-present] +``` + +**Additional details**: + +- If the runner does not support the defined pull policy, the job fails with an error similar to: + `ERROR: Job failed (system failure): the configured PullPolicies ([always]) are not allowed by AllowedPullPolicies ([never])`. + +**Related topics**: + +- [Run your CI/CD jobs in Docker containers](../docker/using_docker_images.md). +- [How runner pull policies work](https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work). +- [Using multiple pull policies](https://docs.gitlab.com/runner/executors/docker.html#using-multiple-pull-policies). + ### `inherit` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207484) in GitLab 12.9. @@ -1945,9 +2013,9 @@ job2: Use `interruptible` if a job should be canceled when a newer pipeline starts before the job completes. -This keyword is used with the [automatic cancellation of redundant pipelines](../pipelines/settings.md#auto-cancel-redundant-pipelines) -feature. When enabled, a running job with `interruptible: true` can be cancelled when -a new pipeline starts on the same branch. +This keyword has no effect if [automatic cancellation of redundant pipelines](../pipelines/settings.md#auto-cancel-redundant-pipelines) +is disabled. When enabled, a running job with `interruptible: true` is cancelled when +starting a pipeline for a new change on the same branch. You can't cancel subsequent jobs after a job with `interruptible: false` starts. @@ -2169,8 +2237,8 @@ build_job: In this example, `build_job` downloads the artifacts from the latest successful `build-1` job on the `main` branch in the `group/project-name` project. -In GitLab 13.3 and later, you can use [CI/CD variables](../variables/index.md) in `needs:project`, -for example: +In GitLab 13.3 and later, you can use [CI/CD variables](../variables/where_variables_can_be_used.md#gitlab-ciyml-file) +in `needs:project`, for example: ```yaml build_job: @@ -2268,43 +2336,58 @@ can use that variable in `needs:pipeline` to download artifacts from the parent To need a job that sometimes does not exist in the pipeline, add `optional: true` to the `needs` configuration. If not defined, `optional: false` is the default. -Jobs that use [`rules`](#rules), [`only`, or `except`](#only--except), might -not always exist in a pipeline. When the pipeline is created, GitLab checks the `needs` -relationships before starting it. Without `optional: true`, needs relationships that -point to a job that does not exist stops the pipeline from starting and causes a pipeline -error similar to: +Jobs that use [`rules`](#rules), [`only`, or `except`](#only--except) might not always +be added to a pipeline. GitLab checks the `needs` relationships before starting a +pipeline: -- `'job1' job needs 'job2' job, but it was not added to the pipeline` +- If the needs entry has `optional: true` and the needed job is present in the pipeline, + the job waits for it to complete before starting. +- If the needed job is not present, the job can start when all other needs requirements are met. +- If the `needs` section contains only optional jobs, and none are added to the pipeline, + the job starts immediately (the same as an empty `needs` entry: `needs: []`). +- If a needed job has `optional: false`, but it was not added to the pipeline, the + pipeline fails to start with an error similar to: `'job1' job needs 'job2' job, but it was not added to the pipeline`. **Keyword type**: Job keyword. You can use it only as part of a job. -**Possible inputs**: - -- `job`: The job to make optional. -- `true` or `false` (default). - **Example of `needs:optional`**: ```yaml -build: +build-job: stage: build + +test-job1: + stage: test + +test-job2: + stage: test rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -rspec: - stage: test +deploy-job: + stage: deploy needs: - - job: build + - job: test-job2 + optional: true + - job: test-job1 + +review-job: + stage: deploy + needs: + - job: test-job2 optional: true ``` In this example: -- When the branch is the default branch, the `build` job exists in the pipeline, and the `rspec` - job waits for it to complete before starting. -- When the branch is not the default branch, the `build` job does not exist in the pipeline. - The `rspec` job runs immediately (similar to `needs: []`) because its `needs` - relationship to the `build` job is optional. +- `build-job`, `test-job1`, and `test-job2` start in stage order. +- When the branch is the default branch, `test-job2` is added to the pipeline, so: + - `deploy-job` waits for both `test-job1` and `test-job2` to complete. + - `review-job` waits for `test-job2` to complete. +- When the branch is not the default branch, `test-job2` is not added to the pipeline, so: + - `deploy-job` waits for only `test-job1` to complete, and does not wait for the missing `test-job2`. + - `review-job` has no other needed jobs and starts immediately (at the same time as `build-job`), + like `needs: []`. #### `needs:pipeline` @@ -2735,7 +2818,7 @@ This example creates a release: **Related topics**: -- [CI/CD example of the `release` keyword](../../user/project/releases/index.md#cicd-example-of-the-release-keyword). +- [CI/CD example of the `release` keyword](../../user/project/releases/index.md#creating-a-release-by-using-a-cicd-job). - [Create multiple releases in a single pipeline](../../user/project/releases/index.md#create-multiple-releases-in-a-single-pipeline). - [Use a custom SSL CA certificate authority](../../user/project/releases/index.md#use-a-custom-ssl-ca-certificate-authority). @@ -2750,7 +2833,9 @@ New tags use the SHA associated with the pipeline. **Possible inputs**: -- A tag name. Can use [CI/CD variables](../variables/index.md). +- A tag name. + +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). **Example of `release:tag_name`**: @@ -2899,7 +2984,7 @@ can be deployed to, but only one deployment can occur per device at any given ti **Possible inputs**: - Only letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces. - It can't start or end with `/`. + It can't start or end with `/`. CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). **Example of `resource_group`**: @@ -3177,8 +3262,9 @@ job: with the flags `File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB`. - For performance reasons, GitLab matches a maximum of 10,000 `exists` patterns or file paths. After the 10,000th check, rules with patterned globs always match. - In other words, the `exists` rule always assumes a match in projects with more - than 10,000 files. + In other words, `exists` always reports `true` if more than 10,000 checks + run. Repositories with less than 10,000 files might still be impacted if the `exists` + rules are checked more than 10,000 times. - `exists` resolves to `true` if any of the listed files are found (an `OR` operation). #### `rules:allow_failure` @@ -3262,6 +3348,8 @@ All jobs except [trigger jobs](#trigger) require a `script` keyword. - Long commands [split over multiple lines](script.md#split-long-commands). - [YAML anchors](yaml_optimization.md#yaml-anchors-for-scripts). +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Example of `script`**: ```yaml @@ -3396,6 +3484,8 @@ to the image specified in the [`image`](#image) keyword. - `<image-name>:<tag>` - `<image-name>@<digest>` +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file), but [not for `alias`](https://gitlab.com/gitlab-org/gitlab/-/issues/19561). + **Example of `services`**: ```yaml @@ -3486,7 +3576,8 @@ Use the `.pre` stage to make a job run at the start of a pipeline. `.pre` is always the first stage in a pipeline. User-defined stages execute after `.pre`. You do not have to define `.pre` in [`stages`](#stages). -You must have a job in at least one stage other than `.pre` or `.post`. +If a pipeline contains only jobs in the `.pre` or `.post` stages, it does not run. +There must be at least one other job in a different stage. **Keyword type**: You can only use it with a job's `stage` keyword. @@ -3521,7 +3612,8 @@ Use the `.post` stage to make a job run at the end of a pipeline. `.post` is always the last stage in a pipeline. User-defined stages execute before `.post`. You do not have to define `.post` in [`stages`](#stages). -You must have a job in at least one stage other than `.pre` or `.post`. +If a pipeline contains only jobs in the `.pre` or `.post` stages, it does not run. +There must be at least one other job in a different stage. **Keyword type**: You can only use it with a job's `stage` keyword. @@ -3566,7 +3658,8 @@ be assigned every tag listed in the job. **Possible inputs**: - An array of tag names. -- [CI/CD variables](../runners/configure_runners.md#use-cicd-variables-in-tags) in GitLab 14.1 and later. +- CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file) + in GitLab 14.1 and later. **Example of `tags`**: @@ -3705,12 +3798,9 @@ successfully complete before starting. #### `trigger:forward` -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213729) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `ci_trigger_forward_variables`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_trigger_forward_variables`. -The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213729) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `ci_trigger_forward_variables`. Disabled by default. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355572) in GitLab 14.10. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/355572) in GitLab 15.1. [Feature flag `ci_trigger_forward_variables`](https://gitlab.com/gitlab-org/gitlab/-/issues/355572) removed. Use `trigger:forward` to specify what to forward to the downstream pipeline. You can control what is forwarded to both [parent-child pipelines](../pipelines/parent_child_pipelines.md) @@ -3779,6 +3869,8 @@ variable defined, the [job-level variable takes precedence](../variables/index.m the first character must be a letter. - The value must be a string. +CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). + **Examples of `variables`**: ```yaml @@ -3844,18 +3936,18 @@ variables: Use `when` to configure the conditions for when jobs run. If not defined in a job, the default value is `when: on_success`. -**Keyword type**: Job keyword. You can use it only as part of a job. +**Keyword type**: Job keyword. You can use it as part of a job. `when: always` and `when: never` can also be used in [`workflow:rules`](#workflow). **Possible inputs**: - `on_success` (default): Run the job only when all jobs in earlier stages succeed or have `allow_failure: true`. - `manual`: Run the job only when [triggered manually](../jobs/job_control.md#create-a-job-that-must-be-run-manually). -- `always`: Run the job regardless of the status of jobs in earlier stages. +- `always`: Run the job regardless of the status of jobs in earlier stages. Can also be used in `workflow:rules`. - `on_failure`: Run the job only when at least one job in an earlier stage fails. - `delayed`: [Delay the execution of a job](../jobs/job_control.md#run-a-job-after-a-delay) for a specified duration. -- `never`: Don't run the job. +- `never`: Don't run the job. Can only be used in a [`rules`](#rules) section or `workflow: rules`. **Example of `when`**: @@ -3921,18 +4013,20 @@ In this example, the script: The following keywords are deprecated. -### Globally-defined `types` +<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> -WARNING: -`types` is deprecated, and is [scheduled to be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/346823). +### Globally-defined `types` (removed) + +The `types` keyword was deprecated in GitLab 9.0, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/346823). Use [`stages`](#stages) instead. -### Job-defined `type` +### Job-defined `type` (removed) -WARNING: -`type` is deprecated, and is [scheduled to be removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/346823). +The `type` keyword was deprecated in GitLab 9.0, and [removed in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/346823). Use [`stage`](#stage) instead. +<!--- end_remove --> + ### Globally-defined `image`, `services`, `cache`, `before_script`, `after_script` Defining `image`, `services`, `cache`, `before_script`, and diff --git a/doc/ci/yaml/yaml_optimization.md b/doc/ci/yaml/yaml_optimization.md index 503ba9bbd80..e5d9e011230 100644 --- a/doc/ci/yaml/yaml_optimization.md +++ b/doc/ci/yaml/yaml_optimization.md @@ -450,5 +450,26 @@ test-vars-2: - printenv ``` -You can't reuse a section that already includes a `!reference` tag. Only one level -of nesting is supported. +### Nest `!reference` tags in `script`, `before_script`, and `after_script` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74792) in GitLab 14.8. + +You can nest `!reference` tags up to 10 levels deep in `script`, `before_script`, and `after_script` sections. Use nested tags to define reusable sections when building more complex scripts. For example: + +```yaml +.snippets: + one: + - echo "ONE!" + two: + - !reference [.snippets, one] + - echo "TWO!" + three: + - !reference [.snippets, two] + - echo "THREE!" + +nested-references: + script: + - !reference [.snippets, three] +``` + +In this example, the `nested-references` job runs all three `echo` commands. diff --git a/doc/cloud_seed/index.md b/doc/cloud_seed/index.md index d4b8d040758..77b22609b12 100644 --- a/doc/cloud_seed/index.md +++ b/doc/cloud_seed/index.md @@ -115,6 +115,6 @@ There are several ways you can contribute to Cloud Seed: and [share feedback](https://gitlab.com/gitlab-org/incubation-engineering/five-minute-production/feedback/-/issues/new?template=general_feedback). - If you are familiar with Ruby on Rails or Vue.js, consider [contributing to GitLab](../development/contributing/index.md) as a developer. - - Much of Cloud Seed is an internal module within the GitLab code base. + - Much of Cloud Seed is an internal module within the GitLab codebase. - If you are familiar with GitLab pipelines, consider contributing to the [Cloud Seed Library](https://gitlab.com/gitlab-org/incubation-engineering/five-minute-production/library) project. diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index 35dbd80e4d1..f524b04c6eb 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -32,14 +32,14 @@ data and are slower to update compared to B-tree indexes. Because of all this one should not blindly add a new index for every column used to filter data by. Instead one should ask themselves the following questions: -1. Can I write my query in such a way that it re-uses as many existing indexes +1. Can you write your query in such a way that it re-uses as many existing indexes as possible? -1. Is the data going to be large enough that using an index will actually be +1. Is the data large enough that using an index is actually faster than just iterating over the rows in the table? 1. Is the overhead of maintaining the index worth the reduction in query timings? -We'll explore every question in detail below. +We explore every question in detail below. ## Re-using Queries @@ -54,7 +54,7 @@ AND state = 'open'; ``` Now imagine we already have an index on the `user_id` column but not on the -`state` column. One may think this query will perform badly due to `state` being +`state` column. One may think this query performs badly due to `state` being unindexed. In reality the query may perform just fine given the index on `user_id` can filter out enough rows. @@ -85,8 +85,8 @@ enough rows you may _not_ want to add a new index. ## Maintenance Overhead Indexes have to be updated on every table write. In case of PostgreSQL _all_ -existing indexes will be updated whenever data is written to a table. As a -result of this having many indexes on the same table will slow down writes. +existing indexes are updated whenever data is written to a table. As a +result of this having many indexes on the same table slows down writes. Because of this one should ask themselves: is the reduction in query performance worth the overhead of maintaining an extra index? @@ -184,8 +184,8 @@ def up end ``` -The call to `index_exists?` will return true if **any** index exists on -`:my_table` and `:my_column`, and index creation will be bypassed. +The call to `index_exists?` returns true if **any** index exists on +`:my_table` and `:my_column`, and index creation is bypassed. The `add_concurrent_index` helper is a requirement for creating indexes on populated tables. Since it cannot be used inside a transactional @@ -285,7 +285,7 @@ production clone. After the index is verified to exist on the production database, create a second merge request that adds the index synchronously. The schema changes must be -updated and committed to `structure.sql` in this second merge request. +updated and committed to `structure.sql` in this second merge request. The synchronous migration results in a no-op on GitLab.com, but you should still add the migration as expected for other installations. The below block demonstrates how to create the second migration for the previous diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md index f5acf0d26eb..51c6e86bb49 100644 --- a/doc/development/adding_service_component.md +++ b/doc/development/adding_service_component.md @@ -23,7 +23,7 @@ The following outline re-uses the [maturity metric](https://about.gitlab.com/dir - [Release management](#release-management) - [Enabled on GitLab.com](feature_flags/controls.md#enabling-a-feature-for-gitlabcom) - Complete - - [Configurable by the GitLab orchestrator](https://gitlab.com/gitlab-org/gitlab-orchestrator) + - [Configurable by the GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) - Lovable - Enabled by default for the majority of users @@ -47,7 +47,7 @@ Adding a new service follows the same [merge request workflow](contributing/merg The first iteration should be to add the ability to connect and use the service as an externally installed component. Often this involves providing settings in GitLab to connect to the service, or allow connections from it. And then shipping documentation on how to install and configure the service with GitLab. -[Elasticsearch](../integration/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives. +[Elasticsearch](../integration/advanced_search/elasticsearch.md#install-elasticsearch) is an example of a service that has been integrated this way. Many of the other services, including internal projects like Gitaly, started off as separately installed alternatives. **For services that depend on the existing GitLab codebase:** diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index f807ed0f85e..de6840b2c6c 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -94,11 +94,16 @@ discussed in [Nullable fields](#nullable-fields). - Lowering the global limits for query complexity and depth. - Anything else that can result in queries hitting a limit that previously was allowed. -Fields that use the [`feature_flag` property](#feature_flag-property) and the flag is disabled by default are exempt -from the deprecation process, and can be removed at any time without notice. - See the [deprecating schema items](#deprecating-schema-items) section for how to deprecate items. +### Breaking change exemptions + +Two scenarios exist where schema items are exempt from the deprecation process, +and can be removed or changed at any time without notice. These are schema items that either: + +- Use the [`feature_flag` property](#feature_flag-property) _and_ the flag is disabled by default. +- Are [marked as alpha](#marking-schema-items-as-alpha). + ## Global IDs The GitLab GraphQL API uses Global IDs (i.e: `"gid://gitlab/MyObject/123"`) @@ -718,6 +723,28 @@ aware of the support. The documentation will mention that the old Global ID style is now deprecated. +## Marking schema items as Alpha + +Fields, arguments, enum values, and mutations can be marked as being in +[alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga). + +An item marked as "alpha" is exempt from the deprecation process and can be removed +at any time without notice. + +This leverages GraphQL deprecations to cause the schema item to appear as deprecated, +and will be described as being in "alpha" in our generated docs and its GraphQL description. + +To mark a schema item as being in "alpha", use the `deprecated:` keyword with `reason: :alpha`. +You must provide the `milestone:` that introduced the alpha item. + +For example: + +```ruby +field :token, GraphQL::Types::String, null: true, + deprecated: { reason: :alpha, milestone: '10.0' }, + description: 'Token for login.' +``` + ## Enums GitLab GraphQL enums are defined in `app/graphql/types`. When defining new enums, the @@ -1848,35 +1875,59 @@ field :created_at, Types::TimeType, null: true, description: 'Timestamp of when ## Testing -### Writing unit tests +For testing mutations and resolvers, consider the unit of +test a full GraphQL request, not a call to a resolver. The reasons for this are +that we want to avoid lots of coupling to the framework, since this makes +upgrades to dependencies much more difficult. -Before creating unit tests, review the following examples: +You should: -- [`spec/graphql/resolvers/users_resolver_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/resolvers/users_resolver_spec.rb) -- [`spec/graphql/mutations/issues/create_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/mutations/issues/create_spec.rb) +- Prefer request specs (either using the full API endpoint or going through + `GitlabSchema.execute`) to unit specs for resolvers and mutations. +- Prefer `GraphqlHelpers#execute_query` and `GraphqlHelpers#run_with_clean_state` to + `GraphqlHelpers#resolve` and `GraphqlHelpers#resolve_field`. -It's faster to test as much of the logic from your GraphQL queries and mutations -with unit tests, which are stored in `spec/graphql`. +For example: -Use unit tests to verify that: +```ruby +# Good: +gql_query = %q(some query text...) +post_graphql(gql_query, current_user: current_user) +# or: +GitlabSchema.execute(gql_query, context: { current_user: current_user }) -- Types have the expected fields. -- Resolvers and mutations apply authorizations and return expected data. -- Edge cases are handled correctly. +# Deprecated: avoid +resolve(described_class, obj: project, ctx: { current_user: current_user }) +``` + +### Writing unit tests (deprecated) + +WARNING: +Avoid writing unit tests if the same thing can be tested with +a full GraphQL request. + +Before creating unit tests, review the following examples: + +- [`spec/graphql/resolvers/users_resolver_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/resolvers/users_resolver_spec.rb) +- [`spec/graphql/mutations/issues/create_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/graphql/mutations/issues/create_spec.rb) ### Writing integration tests Integration tests check the full stack for a GraphQL query or mutation and are stored in `spec/requests/api/graphql`. -For speed, you should test most logic in unit tests instead of integration tests. -However, integration tests that check if data is returned verify the following +For speed, consider calling `GitlabSchema.execute` directly, or making use +of smaller test schemas that only contain the types under test. + +However, full request integration tests that check if data is returned verify the following additional items: - The mutation is actually queryable in the schema (was mounted in `MutationType`). - The data returned by a resolver or mutation correctly matches the [return types](https://graphql-ruby.org/fields/introduction.html#field-return-type) of the fields and resolves without errors. +- The arguments coerce correctly on input, and the fields serialize correctly + on output. Integration tests can also verify the following items, because they invoke the full stack: @@ -1929,6 +1980,55 @@ end ### Testing tips and tricks +- Become familiar with the methods in the `GraphqlHelpers` support module. + Many of these methods make writing GraphQL tests easier. + +- Use traversal helpers like `GraphqlHelpers#graphql_data_at` and + `GraphqlHelpers#graphql_dig_at` to access result fields. For example: + + ```ruby + result = GitlabSchema.execute(query) + + mr_iid = graphql_dig_at(result.to_h, :data, :project, :merge_request, :iid) + ``` + +- Use `GraphqlHelpers#a_graphql_entity_for` to match against results. + For example: + + ```ruby + post_graphql(some_query) + + # checks that it is a hash containing { id => global_id_of(issue) } + expect(graphql_data_at(:project, :issues, :nodes)) + .to contain_exactly(a_graphql_entity_for(issue)) + + # Additional fields can be passed, either as names of methods, or with values + expect(graphql_data_at(:project, :issues, :nodes)) + .to contain_exactly(a_graphql_entity_for(issue, :iid, :title, created_at: some_time)) + ``` + +- Use `GraphqlHelpers#empty_schema` to create an empty schema, rather than creating + one by hand. For example: + + ```ruby + # good + let(:schema) { empty_schema } + + # bad + let(:query_type) { GraphQL::ObjectType.new } + let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)} + ``` + +- Use `GraphqlHelpers#query_double(schema: nil)` of `double('query', schema: nil)`. For example: + + ```ruby + # good + let(:query) { query_double(schema: GitlabSchema) } + + # bad + let(:query) { double('Query', schema: GitlabSchema) } + ``` + - Avoid false positives: Authenticating a user with the `current_user:` argument for `post_graphql` @@ -1983,6 +2083,122 @@ end `spec/requests/api/graphql/ci/pipeline_spec.rb` regardless of the query being used to fetch the pipeline data. +- There can be possible cyclic dependencies within our GraphQL types. + See [Adding field with resolver on a Type causes "Can't determine the return type " error on a different Type](https://github.com/rmosolgo/graphql-ruby/issues/3974#issuecomment-1084444214) + and [Fix unresolved name due to cyclic definition](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84202/diffs#diff-content-32d14251082fd45412e1fdbf5820e62d157e70d2). + + In particular, this can happen with `connection_type`. Normally we might use the following in a resolver: + + ```ruby + type Types::IssueType.connection_type, null: true + ``` + + However this might cause a cyclic definition, which can result in errors like: + + ```ruby + NameError: uninitialized constant Resolvers::GroupIssuesResolver + ``` + + To fix this, we must create a new file that encapsulates the connection type, + and then reference it using double quotes. This gives a delayed resolution, + and the proper connection type. For example: + + ```ruby + module Types + # rubocop: disable Graphql/AuthorizeTypes + class IssueConnectionType < CountableConnectionType + end + end + + Types::IssueConnectionType.prepend_mod_with('Types::IssueConnectionType') + ``` + + in [types/issue_connection_type.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/types/issue_connection_type.rb) + defines a new `Types::IssueConnectionType`, and is then referenced in + [app/graphql/resolvers/base_issues_resolver.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/resolvers/base_issues_resolver.rb) + + ```ruby + type "Types::IssueConnection", null: true + ``` + + Only use this style if you are having spec failures. This is not intended to be a new + pattern that we use. This issue may disappear after we've upgraded to `2.x`. + +- There can be instances where a spec fails because the class is not loaded correctly. + It relates to the + [circular dependencies problem](https://github.com/rmosolgo/graphql-ruby/issues/1929) and + [Adding field with resolver on a Type causes "Can't determine the return type " error on a different Type](https://github.com/rmosolgo/graphql-ruby/issues/3974). + + Unfortunately, the errors generated don't really indicate what the problem is. For example, + remove the quotes from the `Rspec.descrbe` in + [ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb). + Then run `rspec ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb`. + + This generates errors with the expectations. For example: + + ```ruby + 1) Resolvers::ComplianceManagement::MergeRequests::ComplianceViolationResolver#resolve user is authorized filtering the results when given an array of project IDs finds the filtered compliance violations + Failure/Error: expect(subject).to contain_exactly(compliance_violation) + + expected collection contained: [#<MergeRequests::ComplianceViolation id: 4, violating_user_id: 26, merge_request_id: 4, reason: "approved_by_committer", severity_level: "low">] + actual collection contained: [#<MergeRequests::ComplianceViolation id: 4, violating_user_id: 26, merge_request_id: 4, reason: "app...er_id: 27, merge_request_id: 5, reason: "approved_by_merge_request_author", severity_level: "high">] + the extra elements were: [#<MergeRequests::ComplianceViolation id: 5, violating_user_id: 27, merge_request_id: 5, reason: "approved_by_merge_request_author", severity_level: "high">] + # ./ee/spec/graphql/resolvers/compliance_management/merge_requests/compliance_violation_resolver_spec.rb:55:in `block (6 levels) in <top (required)>' + ``` + + However, this is not a case of the wrong result being generated, it's because of the loading order + of the `ComplianceViolationResolver` class. + + The only way we've found to fix this is by quoting the class name in the spec. For example, changing + + ```ruby + RSpec.describe Resolvers::ComplianceManagement::MergeRequests::ComplianceViolationResolver do + ``` + + into: + + ```ruby + RSpec.describe 'Resolvers::ComplianceManagement::MergeRequests::ComplianceViolationResolver' do + ``` + + See [this merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87295#note_946174036) for some discussion. + + Only use this style if you are having spec failures. This is not intended to be a new + pattern that we use. This issue may disappear after we've upgraded to `2.x`. + +- When testing resolvers using `GraphqlHelpers#resolve`, arguments for the resolver can be handled two ways. + + 1. 95% of the resolver specs use arguments that are Ruby objects, as opposed to when using the GraphQL API + only strings and integers are used. This works fine in most cases. + 1. If your resolver takes arguments that use a `prepare` proc, such as a resolver that accepts timeframe + arguments (`TimeFrameArguments`), you must pass the `arg_style: :internal_prepared` parameter into + the `resolve` method. This tells the code to convert the arguments into strings and integers and pass + them through regular argument handling, ensuring that the `prepare` proc is called correctly. + For example in [`iterations_resolver_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/graphql/resolvers/iterations_resolver_spec.rb): + + ```ruby + def resolve_group_iterations(args = {}, obj = group, context = { current_user: current_user }) + resolve(described_class, obj: obj, args: args, ctx: context, arg_style: :internal_prepared) + end + ``` + + One additional caveat is that if you are passing enums as a resolver argument, you must use the + external representation of the enum, rather than the internal. For example: + + ```ruby + # good + resolve_group_iterations({ search: search, in: ['CADENCE_TITLE'] }) + + # bad + resolve_group_iterations({ search: search, in: [:cadence_title] }) + ``` + + The use of `:internal_prepared` was added as a bridge for the + [GraphQL gem](https://graphql-ruby.org) upgrade. Testing resolvers directly will be + [removed eventually](https://gitlab.com/gitlab-org/gitlab/-/issues/363121), + and writing unit tests for resolvers/mutations is + [already deprecated](#writing-unit-tests-deprecated) + ## Notes about Query flow and GraphQL infrastructure The GitLab GraphQL infrastructure can be found in `lib/gitlab/graphql`. diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index c4146b5af3e..6c7213ab235 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -189,3 +189,23 @@ the middleware level, this can be used at the controller or API level. See the `CheckRateLimit` concern for use in controllers. In other parts of the code the `Gitlab::ApplicationRateLimiter` module can be called directly. + +## Next rate limiting architecture + +In May 2022 we've started working on the next iteration of our application +limits framework using a forward looking rate limiting architecture. + +We are working on defining new requirements and designing the next +architecture, so if you need new functionalities to add new limits, instead of +building them right now, consider contributing to the +[Rate Limiting Architecture Working Group](https://about.gitlab.com/company/team/structure/working-groups/rate-limit-architecture/) + +Examples of what features we might want to build into the next iteration of +rate limiting architecture: + +1. Making it possible to define and override limits per namespace / per plan. +1. Automatically generating documentation about what limits are implemented and + what the defaults are. +1. Defining limits in a single place that is easy to find an explore. +1. Soft and hard limits, with support for notifying users when a limit is + approaching. diff --git a/doc/development/application_slis/index.md b/doc/development/application_slis/index.md index 2834723fc01..8d7941865e1 100644 --- a/doc/development/application_slis/index.md +++ b/doc/development/application_slis/index.md @@ -39,8 +39,8 @@ for clarity, they define different metric names: 1. `gitlab_sli:foo_apdex:success_total` for the number of successful measurements. 1. `Gitlab::Metrics::Sli::ErrorRate.new('foo')` defines: - 1. `gitlab_sli:foo_error_rate:total` for the total number of measurements. - 1. `gitlab_sli:foo_error_rate:error_total` for the number of error + 1. `gitlab_sli:foo:total` for the total number of measurements. + 1. `gitlab_sli:foo:error_total` for the number of error measurements - as this is an error rate, it's more natural to talk about errors divided by the total. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 486ef6d27fc..a61a891b096 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -442,9 +442,9 @@ Consul is a tool for service discovery and configuration. Consul is distributed, - [Project page](https://github.com/elastic/elasticsearch/) - Configuration: - - [Omnibus](../integration/elasticsearch.md) - - [Charts](../integration/elasticsearch.md) - - [Source](../integration/elasticsearch.md) + - [Omnibus](../integration/advanced_search/elasticsearch.md) + - [Charts](../integration/advanced_search/elasticsearch.md) + - [Source](../integration/advanced_search/elasticsearch.md) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md) - Layer: Core Service (Data) - GitLab.com: [Get Advanced Search working on GitLab.com (Closed)](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. diff --git a/doc/development/audit_event_guide/index.md b/doc/development/audit_event_guide/index.md index 0d62bcdc3b2..14cd2fd1dc3 100644 --- a/doc/development/audit_event_guide/index.md +++ b/doc/development/audit_event_guide/index.md @@ -14,6 +14,17 @@ new audit events. Audit Events are a tool for GitLab owners and administrators to view records of important actions performed across the application. +## What should not be Audit Events? + +While any events could trigger an Audit Event, not all events should. In general, events that are not good candidates for audit events are: + +- Not attributable to one specific user. +- Not of specific interest to an admin or owner persona. +- Are tracking information for product feature adoption. +- Are covered in the direction page's discussion on [what is not planned](https://about.gitlab.com/direction/manage/compliance/audit-events/#what-is-not-planned-right-now). + +If you have any questions, please reach out to `@gitlab-org/manage/compliance` to see if an Audit Event, or some other approach, may be best for your event. + ## Audit Event Schemas To instrument an audit event, the following attributes should be provided: diff --git a/doc/development/backend/ruby_style_guide.md b/doc/development/backend/ruby_style_guide.md index 6c8125a6157..eff6ae7f217 100644 --- a/doc/development/backend/ruby_style_guide.md +++ b/doc/development/backend/ruby_style_guide.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This is a GitLab-specific style guide for Ruby code. -Generally, if a style is not covered by [existing Rubocop rules or style guides](../contributing/style_guides.md#ruby-rails-rspec), it shouldn't be a blocker. +Generally, if a style is not covered by [existing RuboCop rules or style guides](../contributing/style_guides.md#ruby-rails-rspec), it shouldn't be a blocker. Before adding a new cop to enforce a given style, make sure to discuss it with your team. When the style is approved by a backend EM or by a BE staff eng, add a new section to this page to document the new rule. For every new guideline, add it in a new section and link the discussion from the section's diff --git a/doc/development/build_test_package.md b/doc/development/build_test_package.md index bd2d7545bfc..89b13efc1aa 100644 --- a/doc/development/build_test_package.md +++ b/doc/development/build_test_package.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/cached_queries.md b/doc/development/cached_queries.md index 8c69981b27a..b0bf7c7b6f5 100644 --- a/doc/development/cached_queries.md +++ b/doc/development/cached_queries.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Memory info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -93,7 +93,7 @@ below the query. You can see multiple duplicate cached queries in this modal win  -Click **...** to expand the actual stack trace: +Select **...** to expand the actual stack trace: ```ruby [ diff --git a/doc/development/cascading_settings.md b/doc/development/cascading_settings.md index 76ab2c6e693..56699ff5ffc 100644 --- a/doc/development/cascading_settings.md +++ b/doc/development/cascading_settings.md @@ -210,25 +210,13 @@ This function should be imported and called in the [page-specific JavaScript](fe = s_('Settings|Merge method') .gl-form-radio.custom-control.custom-radio - = f.radio_button :merge_method, :merge, class: "custom-control-input", disabled: merge_method_locked - = f.label :merge_method_merge, class: 'custom-control-label' do - = s_('Settings|Merge commit') - %p.help-text - = s_('Settings|Every merge creates a merge commit.') + = f.gitlab_ui_radio_component :merge_method, :merge, s_('Settings|Merge commit'), help_text: s_('Settings|Every merge creates a merge commit.'), radio_options: { disabled: merge_method_locked } .gl-form-radio.custom-control.custom-radio - = f.radio_button :merge_method, :rebase_merge, class: "custom-control-input", disabled: merge_method_locked - = f.label :merge_method_rebase_merge, class: 'custom-control-label' do - = s_('Settings|Merge commit with semi-linear history') - %p.help-text - = s_('Settings|Every merge creates a merge commit.') + = f.gitlab_ui_radio_component :merge_method, :rebase_merge, s_('Settings|Merge commit with semi-linear history'), help_text: s_('Settings|Every merge creates a merge commit.'), radio_options: { disabled: merge_method_locked } .gl-form-radio.custom-control.custom-radio - = f.radio_button :merge_method, :ff, class: "custom-control-input", disabled: merge_method_locked - = f.label :merge_method_ff, class: 'custom-control-label' do - = s_('Settings|Fast-forward merge') - %p.help-text - = s_('Settings|No merge commits are created.') + = f.gitlab_ui_radio_component :merge_method, :ff, s_('Settings|Fast-forward merge'), help_text: s_('Settings|No merge commits are created.'), radio_options: { disabled: merge_method_locked } = render 'shared/namespaces/cascading_settings/enforcement_checkbox', attribute: :merge_method, diff --git a/doc/development/changelog.md b/doc/development/changelog.md index c19c5b40382..83919bab671 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -11,7 +11,7 @@ file, as well as information and history about our changelog process. ## Overview -Each bullet point, or **entry**, in our +Each list item, or **entry**, in our [`CHANGELOG.md`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/CHANGELOG.md) file is generated from the subject line of a Git commit. Commits are included when they contain the `Changelog` [Git trailer](https://git-scm.com/docs/git-interpret-trailers). diff --git a/doc/development/cicd/img/pipeline_wizard_sample_step1.png b/doc/development/cicd/img/pipeline_wizard_sample_step1.png Binary files differnew file mode 100644 index 00000000000..77e5f07aad2 --- /dev/null +++ b/doc/development/cicd/img/pipeline_wizard_sample_step1.png diff --git a/doc/development/cicd/img/pipeline_wizard_sample_step2.png b/doc/development/cicd/img/pipeline_wizard_sample_step2.png Binary files differnew file mode 100644 index 00000000000..b59414d78f9 --- /dev/null +++ b/doc/development/cicd/img/pipeline_wizard_sample_step2.png diff --git a/doc/development/cicd/img/pipeline_wizard_sample_step3.png b/doc/development/cicd/img/pipeline_wizard_sample_step3.png Binary files differnew file mode 100644 index 00000000000..42c995b64d6 --- /dev/null +++ b/doc/development/cicd/img/pipeline_wizard_sample_step3.png diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index 8677d5b08e3..e8e116037de 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -15,6 +15,12 @@ Development guides that are specific to CI/CD are listed here: See the [CI/CD YAML reference documentation guide](cicd_reference_documentation_guide.md) to learn how to update the [reference page](../../ci/yaml/index.md). +## Examples of CI/CD usage + +We maintain a [`ci-sample-projects`](https://gitlab.com/gitlab-org/ci-sample-projects) group, with projects that showcase +examples of `.gitlab-ci.yml` for different use cases of GitLab CI/CD. They also cover specific syntax that could +be used for different scenarios. + ## CI Architecture overview The following is a simplified diagram of the CI architecture. Some details are left out to focus on diff --git a/doc/development/cicd/pipeline_wizard.md b/doc/development/cicd/pipeline_wizard.md new file mode 100644 index 00000000000..608c21778c0 --- /dev/null +++ b/doc/development/cicd/pipeline_wizard.md @@ -0,0 +1,229 @@ +--- +stage: none +group: Incubation Engineering +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Pipeline Wizard + +The Pipeline Wizard is a Vue frontend component that helps users create a +pipeline by using input fields. The type of input fields and the form of the final +pipeline is configured by a YAML template. + +The Pipeline Wizard expects a single template file that configures the user +flow. The wizard is agnostic with regards to the contents of the file, +so you can use the wizard to display a range of different flows. For example, there +could be one template file for static sites, +one for Docker images, one for mobile apps, and so on. As a first iteration, +these templates are part of the GitLab source code. + +The template file defines multiple steps. The last step shown to the user is always +the commit, and is not part of the template definition. An ideal user experience +consists of 2-3 steps, for a total of 3-4 steps visible to the user. + +## Usage Example + +### Vue Component + +```vue +<!-- ~/my_feature/my_component.vue --> + +<script> + import PipelineWizard from '~/pipeline_wizard/pipeline_wizard.vue' + import template from '~/pipeline_wizard/templates/my_template.yml'; + + export default { + name: "MyComponent", + components: { PipelineWizard }, + data() { + return { template } + }, + methods: { + onDone() { + // redirect + } + } + } +</script> + +<template> + <pipeline-wizard :template="template" + project-path="foo/bar" + default-branch="main" + @done="onDone" /> +</template> +``` + +### Template + +```yaml +# ~/pipeline_wizard/templates/my_template.yml + +title: Set up my specific tech pipeline +description: Here's two or three introductory sentences that help the user understand what this wizard is going to set up. +steps: + # Step 1 + - inputs: + # First input widget + - label: Select your build image + description: A Docker image that we can use to build your image + placeholder: node:lts + widget: text + target: $BUILD_IMAGE + required: true + pattern: "^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-zA-Z0-9-]{1,63}(?<!-))*(?::[0-9]{1,5})?\/)?((?![._-])(?:[a-z0-9._-]*)(?<![._-])(?:\/(?![._-])[a-z0-9._-]*(?<![._-]))*)(?::(?![.-])[a-zA-Z0-9_.-]{1,128})?$" + invalid-feedback: Please enter a valid docker image + + # Second input widget + - label: Installation Steps + description: "Enter the steps that need to run to set up a local build + environment, for example installing dependencies." + placeholder: npm ci + widget: list + target: $INSTALLATION_STEPS + + # This is the template to copy to the final pipeline file and updated with + # the values input by the user. Comments are copied as-is. + template: + my-job: + # The Docker image that will be used to build your app + image: $BUILD_IMAGE + + before_script: $INSTALLATION_STEPS + + artifacts: + paths: + - foo + + # Step 2 + - inputs: + # This is the only input widget for this step + - label: Installation Steps + description: "Enter the steps that need to run to set up a local build + environment, for example installing dependencies." + placeholder: npm ci + widget: list + target: $INSTALLATION_STEPS + + template: + # Functions that should be executed before the build script runs + before_script: $INSTALLATION_STEPS +``` + +### The result + +1.  +1.  +1.  + +### The commit step + +The last step of the wizard is always the commit step. Users can commit the +newly created file to the repository defined by the [wizard's props](#props). +The user has the option to change the branch to commit to. A future iteration +is planned to add the ability to create a MR from here. + +## Component API Reference + +### Props + +- `template` (required): The template content as an unparsed String. See + [Template file location](#template-file-location) for more information. +- `project-path` (required): The full path of the project the final file + should be committed to +- `default-branch` (required): The branch that will be pre-selected during + the commit step. This can be changed by the user. +- `default-filename` (optional, default: `.gitlab-ci.yml`): The Filename + to be used for the file. This can be overridden in the template file. + +### Events + +- `done` - Emitted after the file has been committed. Use this to redirect the +user to the pipeline, for example. + +### Template file location + +Template files are normally stored as YAML files in `~/pipeline_wizard/templates/`. + +The `PipelineWizard` component expects the `template` property as an un-parsed `String`, +and Webpack is configured to load `.yml` files from the above folder as strings. +If you must load the file from a different place, make sure +Webpack does not parse it as an Object. + +## Template Reference + +### Template + +In the root element of the template file, you can define the following properties: + +| Name | Required | Type | Description | +|---------------|------------------------|--------|---------------------------------------------------------------------------------------| +| `title` | **{check-circle}** Yes | string | The page title as displayed to the user. It becomes an `h1` heading above the wizard. | +| `description` | **{check-circle}** Yes | string | The page description as displayed to the user. | +| `filename` | **{dotted-circle}** No | string | The name of the file that is being generated. Defaults to `.gitlab-ci.yml`. | +| `steps` | **{check-circle}** Yes | list | A list of [step definitions](#step-reference). | + +### `step` Reference + +A step makes up one page in a multi-step (or page) process. It consists of one or more +related input fields that build a part of the final `.gitlab-ci.yml`. + +Steps include two properties: + +| Name | Required | Type | Description | +|------------|------------------------|------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `template` | **{check-circle}** Yes | map | The raw YAML to deep-merge into the final `.gitlab-ci.yml`. This template section can contain variables denoted by a `$` sign that is replaced with the values from the input fields. | +| `inputs` | **{check-circle}** Yes | list | A list of [input definitions](#input-reference). | + +### `input` Reference + +Each step can contain one or more `inputs`. For an ideal user experience, it should not +contain more than three. + +The look and feel of the input, as well as the YAML type it produces (string, list, and so on) +depends on the [`widget`](#widgets) used. [`widget: text`](#text) displays a +text input +and inserts the user's input as a string into the template. [`widget: list`](#list) +displays one or more input fields and inserts a list. + +All `inputs` must have a `label`, `widget`, and optionally `target`, but +most properties +are dependent on the widget being used: + +| Name | Required | Type | Description | +|----------|------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------| +| `label` | **{check-circle}** Yes | string | The label for the input field. | +| `widget` | **{check-circle}** Yes | string | The [widget](#widgets) type to use for this input. | +| `target` | **{dotted-circle}** No | string | The variable name inside the step's template that should be replaced with the value of the input field, for example `$FOO`. | + +### Widgets + +#### Text + +Use as `widget: text`. This inserts a `string` in the YAML file. + +| Name | Required | Type | Description | +|-------------------|------------------------|---------|-----------------------| +| `label` | **{check-circle}** Yes | string | The label for the input field. | +| `description` | **{dotted-circle}** No | string | Help text related to the input field. | +| `required` | **{dotted-circle}** No | boolean | Whether or not the user must provide a value before proceeding to the next step. `false` if not defined. | +| `placeholder` | **{dotted-circle}** No | string | A placeholder for the input field. | +| `pattern` | **{dotted-circle}** No | string | A regular expression that the user's input must match before they can proceed to the next step. | +| `invalidFeedback` | **{dotted-circle}** No | string | Help text displayed when the pattern validation fails. | +| `default` | **{dotted-circle}** No | string | The default value for the field. | +| `id` | **{dotted-circle}** No | string | The input field ID is usually autogenerated but can be overridden by providing this property. | + +#### List + +Use as `widget: list`. This inserts a `list` in the YAML file. + +| Name | Required | Type | Description | +|-------------------|------------------------|---------|-----------------------| +| `label` | **{check-circle}** Yes | string | The label for the input field. | +| `description` | **{dotted-circle}** No | string | Help text related to the input field. | +| `required` | **{dotted-circle}** No | boolean | Whether or not the user must provide a value before proceeding to the next step. `false` if not defined. | +| `placeholder` | **{dotted-circle}** No | string | A placeholder for the input field. | +| `pattern` | **{dotted-circle}** No | string | A regular expression that the user's input must match before they can proceed to the next step. | +| `invalidFeedback` | **{dotted-circle}** No | string | Help text displayed when the pattern validation fails. | +| `default` | **{dotted-circle}** No | list | The default value for the list | +| `id` | **{dotted-circle}** No | string | The input field ID is usually autogenerated but can be overridden by providing this property. | diff --git a/doc/development/cicd/schema.md b/doc/development/cicd/schema.md index 0e456a25a7a..ee5b5e4359a 100644 --- a/doc/development/cicd/schema.md +++ b/doc/development/cicd/schema.md @@ -77,7 +77,7 @@ For example, this defines the `retry` keyword: } ] } - } + } } ``` @@ -106,7 +106,7 @@ under the topmost **properties** key. } }, } - } + } } ``` diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 252bd1daf55..a6976271ddf 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -15,35 +15,33 @@ code is effective, understandable, maintainable, and secure. ## Getting your merge request reviewed, approved, and merged -You are strongly encouraged to get your code **reviewed** by a -[reviewer](https://about.gitlab.com/handbook/engineering/workflow/code-review/#reviewer) as soon as -there is any code to review, to get a second opinion on the chosen solution and -implementation, and an extra pair of eyes looking for bugs, logic problems, or -uncovered edge cases. - -The default approach is to choose a reviewer from your group or team for the first review. -This is only a recommendation and the reviewer may be from a different team. -However, it is recommended to pick someone who is a [domain expert](#domain-experts). -If your merge request touches more than one domain (for example, Dynamic Analysis and GraphQL), ask for reviews from an expert from each domain. +Before you begin: -You can read more about the importance of involving reviewers in the section on the responsibility of the author below. +- Familiarize yourself with the [contribution acceptance criteria](contributing/merge_request_workflow.md#contribution-acceptance-criteria). +- If you need some guidance (for example, if it's your first merge request), feel free to ask + one of the [Merge request coaches](https://about.gitlab.com/company/team/?department=merge-request-coach). -If you need some guidance (for example, it's your first merge request), feel free to ask -one of the [Merge request coaches](https://about.gitlab.com/company/team/). +As soon as you have code to review, have the code **reviewed** by a [reviewer](https://about.gitlab.com/handbook/engineering/workflow/code-review/#reviewer). +This reviewer can be from your group or team, or a [domain expert](#domain-experts). +The reviewer can: -If you need assistance with security scans or comments, feel free to include the -Application Security Team (`@gitlab-com/gl-security/appsec`) in the review. +- Give you a second opinion on the chosen solution and implementation. +- Help look for bugs, logic problems, or uncovered edge cases. -Depending on the areas your merge request touches, it must be **approved** by one -or more [maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#maintainer). +For assistance with security scans or comments, include the Application Security Team (`@gitlab-com/gl-security/appsec`). -For approvals, we use the approval functionality found in the merge request -widget. For reviewers, we use the [reviewer functionality](../user/project/merge_requests/getting_started.md#reviewer) in the sidebar. +The reviewers use the [reviewer functionality](../user/project/merge_requests/getting_started.md#reviewer) in the sidebar. Reviewers can add their approval by [approving additionally](../user/project/merge_requests/approvals/index.md#approve-a-merge-request). +Depending on the areas your merge request touches, it must be **approved** by one +or more [maintainers](https://about.gitlab.com/handbook/engineering/workflow/code-review/#maintainer). +The **Approved** button is in the merge request widget. + Getting your merge request **merged** also requires a maintainer. If it requires more than one approval, the last maintainer to review and approve merges it. +Read more about [author responsibilities](#the-responsibility-of-the-merge-request-author) below. + ### Domain experts Domain experts are team members who have substantial experience with a specific technology, @@ -90,7 +88,7 @@ page, with these behaviors: 1. People whose [GitLab status](../user/profile/index.md#set-your-current-status) emoji is 🔶 `:large_orange_diamond:` or 🔸 `:small_orange_diamond:` are half as likely to be picked. 1. It always picks the same reviewers and maintainers for the same - branch name (unless their out-of-office (OOO) status changes, as in point 1). It + branch name (unless their out-of-office (`OOO`) status changes, as in point 1). It removes leading `ce-` and `ee-`, and trailing `-ce` and `-ee`, so that it can be stable for backport branches. @@ -110,14 +108,14 @@ As described in the section on the responsibility of the maintainer below, you are recommended to get your merge request approved and merged by maintainers with [domain expertise](#domain-experts). -1. If your merge request includes backend changes (*1*), it must be +1. If your merge request includes `~backend` changes (*1*), it must be **approved by a [backend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_backend)**. 1. If your merge request includes database migrations or changes to expensive queries (*2*), it must be **approved by a [database maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_database)**. Read the [database review guidelines](database_review.md) for more details. -1. If your merge request includes frontend changes (*1*), it must be +1. If your merge request includes `~frontend` changes (*1*), it must be **approved by a [frontend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_frontend)**. -1. If your merge request includes user-facing changes (*3*), it must be +1. If your merge request includes (`~UX`) user-facing changes (*3*), it must be **approved by a [Product Designer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_UX)**. See the [design and user interface guidelines](contributing/design.md) for details. 1. If your merge request includes adding a new JavaScript library (*1*)... @@ -143,7 +141,7 @@ with [domain expertise](#domain-experts). 1. If your merge request introduces a new service to GitLab (Puma, Sidekiq, Gitaly are examples), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. See the [process for adding a service component to GitLab](adding_service_component.md) for details. 1. If your merge request includes changes related to authentication or authorization, it must be **approved by a [Manage:Authentication and Authorization team member](https://about.gitlab.com/company/team/)**. Check the [code review section on the group page](https://about.gitlab.com/handbook/engineering/development/dev/manage/authentication-and-authorization/#additional-considerations) for more details. Patterns for files known to require review from the team are listed in the in the `Authentication and Authorization` section of the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS) file, and the team will be listed in the approvers section of all merge requests that modify these files. -- (*1*): Specs other than JavaScript specs are considered backend code. +- (*1*): Specs other than JavaScript specs are considered `~backend` code. Haml markup is considered `~frontend` code. However, Ruby code within Haml templates is considered `~backend` code. - (*2*): We encourage you to seek guidance from a database maintainer if your merge request is potentially introducing expensive queries. It is most efficient to comment on the line of code in question with the SQL queries so they can give their advice. @@ -185,6 +183,7 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering ##### Observability instrumentation 1. I have included enough instrumentation to facilitate debugging and proactive performance improvements through observability. + See [example](https://gitlab.com/gitlab-org/gitlab/-/issues/346124#expectations) of adding feature flags, logging, and instrumentation. ##### Documentation @@ -237,6 +236,8 @@ up confusion or verify that the end result matches what they had in mind, to database specialists to get input on the data model or specific queries, or to any other developer to get an in-depth review of the solution. +If your merge request touches more than one domain (for example, Dynamic Analysis and GraphQL), ask for reviews from an expert from each domain. + If an author is unsure if a merge request needs a [domain expert's](#domain-experts) opinion, then that indicates it does. Without it, it's unlikely they have the required level of confidence in their solution. @@ -246,7 +247,7 @@ request diff alerting the reviewer to anything important as well as for anything that demands further explanation or attention. Examples of content that may warrant a comment could be: -- The addition of a linting rule (Rubocop, JS etc). +- The addition of a linting rule (RuboCop, JS etc). - The addition of a library (Ruby gem, JS lib etc). - Where not obvious, a link to the parent class or method. - Any benchmarking performed to complement the change. @@ -269,10 +270,18 @@ This saves reviewers time and helps authors catch mistakes earlier. ### The responsibility of the reviewer -[Review the merge request](#reviewing-a-merge-request) thoroughly. When you are confident +[Review the merge request](#reviewing-a-merge-request) thoroughly. + +Verify that the merge request meets all [contribution acceptance criteria](contributing/merge_request_workflow.md#contribution-acceptance-criteria). + +If a merge request is too large, fixes more than one issue, or implements more +than one feature, you should guide the author towards spltting the merge request +into smaller merge requests. + +When you are confident that it meets all requirements, you should: -- Click the Approve button. +- Select **Approve**. - `@` mention the author to generate a to-do notification, and advise them that their merge request has been reviewed and approved. - Request a review from a maintainer. Default to requests for a maintainer with [domain expertise](#domain-experts), however, if one isn't available or you think the merge request doesn't need a review by a [domain expert](#domain-experts), feel free to follow the [Reviewer roulette](#reviewer-roulette) suggestion. @@ -291,6 +300,12 @@ Because a maintainer's job only depends on their knowledge of the overall GitLab codebase, and not that of any specific domain, they can review, approve, and merge merge requests from any team and in any product area. +If a merge request is too large, fixes more than one issue, or implements more +than one feature, the maintainer can ask the author to make the merge request +smaller. Request the previous reviewer, or a merge request coach to help guide +the author on how to split the merge request, and to review the resulting +changes. + Maintainers do their best to also review the specifics of the chosen solution before merging, but as they are not necessarily [domain experts](#domain-experts), they may be poorly placed to do so without an unreasonable investment of time. In those cases, they diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index 7f5c800216a..ce013a9254b 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -54,7 +54,7 @@ Check visual design properties using your browser's _elements inspector_ ([Chrom - Use recommended [colors](https://design.gitlab.com/product-foundations/colors/) and [typography](https://design.gitlab.com/product-foundations/type-fundamentals/). - Follow [layout guidelines](https://design.gitlab.com/layout/grid/). -- Use existing [icons](http://gitlab-org.gitlab.io/gitlab-svgs/) and [illustrations](http://gitlab-org.gitlab.io/gitlab-svgs/illustrations/) +- Use existing [icons](https://gitlab-org.gitlab.io/gitlab-svgs/) and [illustrations](https://gitlab-org.gitlab.io/gitlab-svgs/illustrations/) or propose new ones according to [iconography](https://design.gitlab.com/product-foundations/iconography/) and [illustration](https://design.gitlab.com/product-foundations/illustration/) guidelines. @@ -98,7 +98,7 @@ Check accessibility using your browser's _accessibility inspector_ ([Chrome](htt When the design is ready, _before_ starting its implementation: -- Share design specifications in the related issue, preferably through a [Figma link](https://help.figma.com/hc/en-us/articles/360040531773-Share-Files-with-anyone-using-Link-Sharing#Copy_links) +- Share design specifications in the related issue, preferably through a [Figma link](https://help.figma.com/hc/en-us/articles/360040531773-Share-Files-with-anyone-using-Link-Sharing#copy-link) link or [GitLab Designs feature](../../user/project/issues/design_management.md). See [when you should use each tool](https://about.gitlab.com/handbook/engineering/ux/product-designer/#deliver). - Document user flow and states (for example, using [Mermaid flowcharts in Markdown](../../user/markdown.md#mermaid)). diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 8a4b06840a4..182d00d52ab 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -37,7 +37,7 @@ Report suspected security vulnerabilities by following the [disclosure process on the GitLab.com website](https://about.gitlab.com/security/disclosure/). WARNING: -Do **NOT** create publicly viewable issues for suspected security vulnerabilities. +Do **not** create publicly viewable issues for suspected security vulnerabilities. ## Code of conduct @@ -128,9 +128,12 @@ The general flow of contributing to GitLab is: 1. In the merge request's description: - Ensure you provide complete and accurate information. - Review the provided checklist. -1. Assign the merge request (if possible) to, or [mention](../../user/discussions/index.md#mentions), - one of the [code owners](../../user/project/code_owners.md) for the relevant project, - and explain that you are ready for review. +1. Once you're ready, mark your MR as ready for review with `@gitlab-bot ready`. + - This will add the `~"workflow::ready for review"` label, and then automatically assign a merge request coach as reviewer. + - If you know a relevant reviewer (for example, someone that was involved a related issue), you can also + assign them directly with `@gitlab-bot ready @username`. + +#### Review process When you submit code to GitLab, we really want it to get merged! However, we always review submissions carefully, and this takes time. Code submissions will usually be reviewed by two @@ -139,7 +142,11 @@ submissions carefully, and this takes time. Code submissions will usually be rev - A [reviewer](../code_review.md#the-responsibility-of-the-reviewer). - A [maintainer](../code_review.md#the-responsibility-of-the-maintainer). -Keep the following in mind when submitting merge requests: +After review, the reviewer could ask the author to update the merge request. In that case, the reviewer would set the `~"workflow::in dev"` label. +Once the merge request has been updated and set as ready for review again (for example, with `@gitlab-bot ready`), they will review the code again. +This process may repeat any number of times before merge, to help make the contribution the best it can be. + +Lastly, keep the following in mind when submitting merge requests: - When reviewers are reading through a merge request they may request guidance from other reviewers. @@ -154,13 +161,11 @@ Keep the following in mind when submitting merge requests: [approval](../../user/project/merge_requests/approvals/index.md) of merge requests, the maintainer may require [approvals from certain reviewers](../code_review.md#approval-guidelines) before merging a merge request. -- After review, the author may be asked to update the merge request. Once the merge request has been - updated and reassigned to the reviewer, they will review the code again. This process may repeat - any number of times before merge, to help make the contribution the best it can be. +- Sometimes a maintainer may choose to close a merge request. They will fully disclose why it will not + be merged, as well as some guidance. The maintainers will be open to discussion about how to change + the code so it can be approved and merged in the future. -Sometimes a maintainer may choose to close a merge request. They will fully disclose why it will not -be merged, as well as some guidance. The maintainers will be open to discussion about how to change -the code so it can be approved and merged in the future. +#### Getting attention on your merge request GitLab will do its best to review community contributions as quickly as possible. Specially appointed developers review community contributions daily. Look at the @@ -170,8 +175,9 @@ written some front-end code, you should mention the frontend merge request coach your code has multiple disciplines, you may mention multiple merge request coaches. GitLab receives a lot of community contributions. If your code has not been reviewed within two -working days of its initial submission, feel free to mention all merge request coaches with -`@gitlab-org/coaches` to get their attention. +working days of its initial submission, you can ask for help with `@gitlab-bot help`. + +#### Addition of external libraries When submitting code to GitLab, you may feel that your contribution requires the aid of an external library. If your code includes an external library, please provide a link to the library, as well as diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index ee1ed744cd4..eff1d2e671d 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -281,7 +281,7 @@ requirements. 1. The change is tested in a review app where possible and if appropriate. 1. The new feature does not degrade the user experience of the product. 1. The change is evaluated to [limit the impact of far-reaching work](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work). -1. An agreed-upon [rollout plan](https://about.gitlab.com/handbook/engineering/development/processes/rollout-plans/). +1. An agreed-upon [rollout plan](https://about.gitlab.com/handbook/engineering/development/processes/rollout-plans/). 1. Merged by a project maintainer. ### Production use @@ -292,7 +292,7 @@ requirements. 1. If there is a performance risk in the change, I have analyzed the performance of the system before and after the change. 1. *If the merge request uses feature flags, per-project or per-group enablement, and a staged rollout:* - Confirmed to be working on GitLab projects. - - Confirmed to be working at each stage for all projects added. + - Confirmed to be working at each stage for all projects added. 1. Added to the [release post](https://about.gitlab.com/handbook/marketing/blog/release-posts/), if relevant. 1. Added to [the website](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/features.yml), if relevant. @@ -322,7 +322,7 @@ issue) that are incremental improvements, such as: 1. Unprioritized bug fixes (for example, [Banner alerting of project move is showing up everywhere](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18985)) 1. Documentation improvements -1. Rubocop or Code Quality improvements +1. RuboCop or Code Quality improvements Tag a merge request with ~"Stuff that should Just Work" to track work in this area. diff --git a/doc/development/contributing/verify/index.md b/doc/development/contributing/verify/index.md index 01aacffd00f..09b206d59aa 100644 --- a/doc/development/contributing/verify/index.md +++ b/doc/development/contributing/verify/index.md @@ -53,7 +53,7 @@ and they serve us and our users well. Some examples of these principles are that - The feedback delivered by GitLab CI/CD and data produced by the platform should be accurate. If a job fails and we notify a user that it was successful, it can have severe negative consequences. - Feedback needs to be available when a user needs it and data can not disappear unexpectedly when engineers need it. -- It all doesn’t matter if the platform is not secure and we +- It all doesn't matter if the platform is not secure and we are leaking credentials or secrets. - When a user provides a set of preconditions in a form of CI/CD configuration, the result should be deterministic each time a pipeline runs, because otherwise the platform might not be trustworthy. - If it is fast, simple to use and has a great UX it will serve our users well. @@ -134,7 +134,7 @@ applied to many other technical implementations. GitLab is a DevOps platform. We popularize DevOps because it helps companies be more efficient and achieve better results. One important component of DevOps culture is to take ownership over features and code that you are -building. It is very difficult to do that when you don’t know how your features +building. It is very difficult to do that when you don't know how your features perform and behave in the production environment. This is why we want to make our features and code observable. It @@ -164,15 +164,29 @@ data from the database, file system, or object storage, you should get an extra of eyes on your changes. When you are defining a new retention policy, you should double check with PMs and EMs. +### Get your design reviewed + +When you are designing a subsystem for pipeline processing and transitioning +CI/CD statuses, request an additional opinion on the design from a Verify maintainer (`@gitlab-org/maintainers/cicd-verify`) +as early as possible and hold others accountable for doing the same. Having your +design reviewed by a Verify maintainer helps to identify any blind spots you might +have overlooked as early as possible and possibly leads to a better solution. + +By having the design reviewed before any development work is started, it also helps to +make merge request review more efficient. You would be less likely to encounter +significantly differing opinions or change requests during the maintainer review +if the design has been reviewed by a Verify maintainer. As a result, the merge request +could be merged sooner. + ### Get your changes reviewed -When your merge request is ready for reviews you must assign -reviewers and then maintainers. Depending on the complexity of a change, you -might want to involve the people that know the most about the codebase area you are -changing. We do have many domain experts in Verify and it is absolutely acceptable to -ask them to review your code when you are not certain if a reviewer or -maintainer assigned by the Reviewer Roulette has enough context about the -change. +When your merge request is ready for reviews you must assign reviewers and then +maintainers. Depending on the complexity of a change, you might want to involve +the people that know the most about the codebase area you are changing. We do +have many domain experts and maintainers in Verify and it is absolutely +acceptable to ask them to review your code when you are not certain if a +reviewer or maintainer assigned by the Reviewer Roulette has enough context +about the change. The reviewer roulette offers useful suggestions, but as assigning the right reviewers is important it should not be done automatically every time. It might @@ -181,9 +195,19 @@ updating, because their feedback might be limited to code style and syntax. Depending on the complexity and impact of a change, assigning the right people to review your changes might be very important. -If you don’t know who to assign, consult `git blame` or ask in the `#verify` +If you don't know who to assign, consult `git blame` or ask in the `#s_verify` Slack channel (GitLab team members only). +There are two kinds of changes / merge requests that require additional +attention from reviews and an additional reviewer: + +1. Merge requests changing code around pipelines / stages / builds statuses. +1. Merge requests changing code around authentication / security features. + +In both cases engineers are expected to request a review from a maintainer and +a domain expert. If maintainer is the domain expert, involving another person +is recommended. + ### Incremental rollouts After your merge request is merged by a maintainer, it is time to release it to @@ -220,7 +244,7 @@ scenario relating to a software being built by one of our [early customers](http That would be quite an undesirable outcome of a small bug in GitLab CI/CD status processing. Please take extra care when you are working on CI/CD statuses, -we don’t want to implode our Universe! +we don't want to implode our Universe! This is an extreme and unlikely scenario, but presenting data that is not accurate can potentially cause a myriad of problems through the @@ -230,6 +254,22 @@ can have disastrous consequences. GitLab CI/CD is being used by companies building medical, aviation, and automotive software. Continuous Integration is a mission critical part of software engineering. -When you are working on a subsystem for pipeline processing and transitioning -CI/CD statuses, request an additional opinion on the design from a domain expert -as early as possible and hold others accountable for doing the same. +### Definition of Done + +In Verify, we follow our Development team's [Definition of Done](../merge_request_workflow.md#definition-of-done). +We also want to keep things efficient and [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) when we answer questions +and solve problems for our users. + +For any issue that is resolved because the solution is supported with existing `.gitlab-ci.yml` syntax, +create a project in the [`ci-sample-projects`](https://gitlab.com/gitlab-org/ci-sample-projects) group +that demonstrates the solution. + +The project must have: + +- A simple title. +- A clear description. +- A `README.md` with: + - A link to the resolved issue. You should also direct users to collaborate in the + resolved issue if any questions arise. + - A link to any relevant documentation. + - A detailed explanation of what the example is doing. diff --git a/doc/development/creating_enums.md b/doc/development/creating_enums.md index 1f04f4c9712..450cb97d978 100644 --- a/doc/development/creating_enums.md +++ b/doc/development/creating_enums.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -98,7 +98,7 @@ end This looks working as a workaround, however, this approach has some downsides that: - Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. - For example, when you move `activity_limit_exceeded` to FOSS, you'll see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. + For example, when you move `activity_limit_exceeded` to FOSS, you see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. - The integer column for the `enum` is likely created [as `SMALLINT`](#creating-enums). Therefore, you need to be careful of that the offset doesn't exceed the maximum value of 2 bytes integer. diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index bfd455ef9da..9842814816f 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index 3cf9ab1ab5c..2d079656e23 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -15,7 +15,7 @@ requiring downtime. ## Dropping Columns Removing columns is tricky because running GitLab processes may still be using -the columns. To work around this safely, you will need three steps in three releases: +the columns. To work around this safely, you need three steps in three releases: 1. Ignoring the column (release M) 1. Dropping the column (release M+1) @@ -77,7 +77,7 @@ bundle exec rails g post_deployment_migration remove_users_updated_at_column There are two scenarios that you need to consider to write a migration that removes a column: -#### A. The removed column has no indexes or constraints that belong to it +#### A. The removed column has no indexes or constraints that belong to it In this case, a **transactional migration** can be used. Something as simple as: @@ -170,12 +170,12 @@ class RenameUsersUpdatedAtToUpdatedAtTimestamp < Gitlab::Database::Migration[1.0 end ``` -This will take care of renaming the column, ensuring data stays in sync, and +This takes care of renaming the column, ensuring data stays in sync, and copying over indexes and foreign keys. If a column contains one or more indexes that don't contain the name of the -original column, the previously described procedure will fail. In that case, -you'll first need to rename these indexes. +original column, the previously described procedure fails. In that case, +you need to rename these indexes. ### Step 2: Add A Post-Deployment Migration @@ -270,7 +270,7 @@ And that's it, we're done! Some type changes require casting data to a new type. For example when changing from `text` to `jsonb`. In this case, use the `type_cast_function` option. -Make sure there is no bad data and the cast will always succeed. You can also provide a custom function that handles +Make sure there is no bad data and the cast always succeeds. You can also provide a custom function that handles casting errors. Example migration: @@ -291,8 +291,9 @@ They can also produce a lot of pressure on the database due to it rapidly updating many rows in sequence. To reduce database pressure you should instead use a background migration -when migrating a column in a large table (for example, `issues`). This will -spread the work / load over a longer time period, without slowing down deployments. +when migrating a column in a large table (for example, `issues`). Background +migrations spread the work / load over a longer time period, without slowing +down deployments. For more information, see [the documentation on cleaning up background migrations](background_migrations.md#cleaning-up). @@ -533,7 +534,7 @@ step approach: Usually this works, but not always. For example, if a field's format is to be changed from JSON to something else we have a bit of a problem. If we were to -change existing data before deploying application code we'll most likely run +change existing data before deploying application code we would most likely run into errors. On the other hand, if we were to migrate after deploying the application code we could run into the same problems. diff --git a/doc/development/database/background_migrations.md b/doc/development/database/background_migrations.md index 80ba0336bda..0124dbae51f 100644 --- a/doc/development/database/background_migrations.md +++ b/doc/development/database/background_migrations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -65,7 +65,7 @@ and idempotent. See [Sidekiq best practices guidelines](https://github.com/mperham/sidekiq/wiki/Best-Practices) for more details. -Make sure that in case that your migration job is going to be retried data +Make sure that in case that your migration job is retried, data integrity is guaranteed. ## Background migrations for EE-only features @@ -77,7 +77,7 @@ as explained in the [guidelines for implementing Enterprise Edition features](.. ## How It Works Background migrations are simple classes that define a `perform` method. A -Sidekiq worker will then execute such a class, passing any arguments to it. All +Sidekiq worker then executes such a class, passing any arguments to it. All migration classes must be defined in the namespace `Gitlab::BackgroundMigration`, the files should be placed in the directory `lib/gitlab/background_migration/`. @@ -100,13 +100,13 @@ to automatically split the job into batches: ```ruby queue_background_migration_jobs_by_range_at_intervals( ClassName, - BackgroundMigrationClassName, + 'BackgroundMigrationClassName', 2.minutes, batch_size: 10_000 ) ``` -You'll also need to make sure that newly created data is either migrated, or +You also need to make sure that newly created data is either migrated, or saved in both the old and new version upon creation. For complex and time consuming migrations it's best to schedule a background job using an `after_create` hook so this doesn't affect response timings. The same applies to @@ -142,7 +142,7 @@ or minor release, you _must not_ do this in a patch release. Because background migrations can take a long time you can't immediately clean things up after scheduling them. For example, you can't drop a column that's used in the migration process as this would cause jobs to fail. This means that -you'll need to add a separate _post deployment_ migration in a future release +you need to add a separate _post deployment_ migration in a future release that finishes any remaining jobs before cleaning things up (for example, removing a column). @@ -189,7 +189,7 @@ extract the `url` key from this JSON object and store it in the `integrations.ur column. There are millions of integrations and parsing JSON is slow, thus you can't do this in a regular migration. -To do this using a background migration we'll start with defining our migration +To do this using a background migration we start with defining our migration class: ```ruby @@ -213,7 +213,7 @@ class Gitlab::BackgroundMigration::ExtractIntegrationsUrl end ``` -Next we'll need to adjust our code so we schedule the above migration for newly +Next we need to adjust our code so we schedule the above migration for newly created and updated integrations. We can do this using something along the lines of the following: @@ -232,7 +232,7 @@ We're using `after_commit` here to ensure the Sidekiq job is not scheduled before the transaction completes as doing so can lead to race conditions where the changes are not yet visible to the worker. -Next we'll need a post-deployment migration that schedules the migration for +Next we need a post-deployment migration that schedules the migration for existing data. ```ruby @@ -254,11 +254,11 @@ class ScheduleExtractIntegrationsUrl < Gitlab::Database::Migration[1.0] end ``` -Once deployed our application will continue using the data as before but at the -same time will ensure that both existing and new data is migrated. +After deployed our application continues using the data as before, but at the +same time ensures that both existing and new data is migrated. In the next release we can remove the `after_commit` hooks and related code. We -will also need to add a post-deployment migration that consumes any remaining +also need to add a post-deployment migration that consumes any remaining jobs and manually run on any un-migrated rows. Such a migration would look like this: @@ -292,7 +292,7 @@ If the application does not depend on the data being 100% migrated (for instance, the data is advisory, and not mission-critical), then this final step can be skipped. -This migration will then process any jobs for the ExtractIntegrationsUrl migration +This migration then processes any jobs for the `ExtractIntegrationsUrl` migration and continue once all jobs have been processed. Once done you can safely remove the `integrations.properties` column. @@ -325,13 +325,13 @@ for more details. 1. Make sure that tests you write are not false positives. 1. Make sure that if the data being migrated is critical and cannot be lost, the clean-up migration also checks the final state of the data before completing. -1. When migrating many columns, make sure it won't generate too many +1. When migrating many columns, make sure it does not generate too many dead tuples in the process (you may need to directly query the number of dead tuples and adjust the scheduling according to this piece of data). 1. Make sure to discuss the numbers with a database specialist, the migration may add more pressure on DB than you expect (measure on staging, or ask someone to measure on production). -1. Make sure to know how much time it'll take to run all scheduled migrations. +1. Make sure to know how much time it takes to run all scheduled migrations. 1. Provide an estimation section in the description, estimating both the total migration run time and the query times for each background migration job. Explain plans for each query should also be provided. @@ -503,6 +503,6 @@ View the production Sidekiq log and filter for: - `json.meta.caller_id: <MyBackgroundMigrationSchedulingMigrationClassName>` - `json.args: <MyBackgroundMigrationClassName>` -Looking at the `json.error_class`, `json.error_message` and `json.error_backtrace` values may be helpful in understanding why the jobs failed. +Looking at the `json.exception.class`, `json.exception.message`, `json.exception.backtrace`, and `json.exception.sql` values may be helpful in understanding why the jobs failed. Depending on when and how the failure occurred, you may find other helpful information by filtering with `json.class: <MyBackgroundMigrationClassName>`. diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index 3a0fa77eff9..6d3d5fa7f92 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -1,6 +1,6 @@ --- type: reference, dev -stage: Enablement +stage: Data Stores group: Database info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" --- @@ -152,9 +152,7 @@ When you start the second post-deployment migration, delete the previously batched migration with the provided code: ```ruby -Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION_NAME, TABLE_NAME, COLUMN, JOB_ARGUMENTS) - .delete_all +delete_batched_background_migration(MIGRATION_NAME, TABLE_NAME, COLUMN, JOB_ARGUMENTS) ``` ## Cleaning up @@ -192,7 +190,7 @@ data to be in the new format. The `routes` table has a `source_type` field that's used for a polymorphic relationship. As part of a database redesign, we're removing the polymorphic relationship. One step of -the work will be migrating data from the `source_id` column into a new singular foreign key. +the work is migrating data from the `source_id` column into a new singular foreign key. Because we intend to delete old rows later, there's no need to update them as part of the background migration. @@ -221,9 +219,9 @@ background migration. NOTE: Job classes must be subclasses of `BatchedMigrationJob` to be correctly handled by the batched migration framework. Any subclass of - `BatchedMigrationJob` will be initialized with necessary arguments to + `BatchedMigrationJob` is initialized with necessary arguments to execute the batch, as well as a connection to the tracking database. - Additional `job_arguments` set on the migration will be passed to the + Additional `job_arguments` set on the migration are passed to the job's `perform` method. 1. Add a new trigger to the database to update newly created and updated routes, @@ -245,12 +243,14 @@ background migration. 1. Create a post-deployment migration that queues the migration for existing data: ```ruby - class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[1.0] + class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[2.0] disable_ddl_transaction! MIGRATION = 'BackfillRouteNamespaceId' DELAY_INTERVAL = 2.minutes + restrict_gitlab_migration gitlab_schema: :gitlab_main + def up queue_batched_background_migration( MIGRATION, @@ -261,12 +261,19 @@ background migration. end def down - Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(MIGRATION, :routes, :id, []).delete_all + delete_batched_background_migration(MIGRATION, :routes, :id, []) end end ``` + NOTE: + When queuing a batched background migration, you need to restrict + the schema to the database where you make the actual changes. + In this case, we are updating `routes` records, so we set + `restrict_gitlab_migration gitlab_schema: :gitlab_main`. If, however, + you need to perform a CI data migration, you would set + `restrict_gitlab_migration gitlab_schema: :gitlab_ci`. + After deployment, our application: - Continues using the data as before. - Ensures that both existing and new data are migrated. @@ -275,16 +282,19 @@ background migration. that checks that the batched background migration is completed. For example: ```ruby - class FinalizeBackfillRouteNamespaceId < Gitlab::Database::Migration[1.0] + class FinalizeBackfillRouteNamespaceId < Gitlab::Database::Migration[2.0] MIGRATION = 'BackfillRouteNamespaceId' disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + def up ensure_batched_background_migration_is_finished( job_class_name: MIGRATION, table_name: :routes, column_name: :id, - job_arguments: [] + job_arguments: [], + finalize: true ) end @@ -294,6 +304,11 @@ background migration. end ``` + NOTE: + If the batched background migration is not finished, the system will + execute the batched background migration inline. If you don't want + to see this behavior, you need to pass `finalize: false`. + If the application does not depend on the data being 100% migrated (for instance, the data is advisory, and not mission-critical), then you can skip this final step. This step confirms that the migration is completed, and all of the rows were migrated. diff --git a/doc/development/database/client_side_connection_pool.md b/doc/development/database/client_side_connection_pool.md index 60c8665df87..dc52a551407 100644 --- a/doc/development/database/client_side_connection_pool.md +++ b/doc/development/database/client_side_connection_pool.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/constraint_naming_convention.md b/doc/development/database/constraint_naming_convention.md index a22ddc1551c..72f16c20559 100644 --- a/doc/development/database/constraint_naming_convention.md +++ b/doc/development/database/constraint_naming_convention.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/database_lab.md b/doc/development/database/database_lab.md index 1c8694b113d..5346df2690d 100644 --- a/doc/development/database/database_lab.md +++ b/doc/development/database/database_lab.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/database_migration_pipeline.md b/doc/development/database/database_migration_pipeline.md index ce7e1801abc..496bd09bf1d 100644 --- a/doc/development/database/database_migration_pipeline.md +++ b/doc/development/database/database_migration_pipeline.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index ca9ca36b156..b6bbfe690c1 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -47,7 +47,7 @@ As a database reviewer, join the internal `#database` Slack channel and ask ques database related issues with other database reviewers and maintainers. There is also an optional database office hours call held bi-weekly, alternating between -European/US and APAC friendly hours. You can join the office hours call and bring topics +European/US and Asia-Pacific (APAC) friendly hours. You can join the office hours call and bring topics that require a more in-depth discussion between the database reviewers and maintainers: - [Database Office Hours Agenda](https://docs.google.com/document/d/1wgfmVL30F8SdMg-9yY6Y8djPSxWNvKmhR5XmsvYX1EI/edit). diff --git a/doc/development/database/dbcheck-migrations-job.md b/doc/development/database/dbcheck-migrations-job.md index af72e28a875..49f8b183272 100644 --- a/doc/development/database/dbcheck-migrations-job.md +++ b/doc/development/database/dbcheck-migrations-job.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/deleting_migrations.md b/doc/development/database/deleting_migrations.md index be9009f365d..8354cb62d0c 100644 --- a/doc/development/database/deleting_migrations.md +++ b/doc/development/database/deleting_migrations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/efficient_in_operator_queries.md b/doc/development/database/efficient_in_operator_queries.md index 2503be826ea..a2481577e8c 100644 --- a/doc/development/database/efficient_in_operator_queries.md +++ b/doc/development/database/efficient_in_operator_queries.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -26,7 +26,7 @@ Pagination may be used to fetch subsequent records. Example tasks requiring querying nested domain objects from the group level: - Show first 20 issues by creation date or due date from the group `gitlab-org`. -- Show first 20 merge_requests by merged at date from the group `gitlab-com`. +- Show first 20 merge requests by merged at date from the group `gitlab-com`. Unfortunately, ordered group-level queries typically perform badly as their executions require heavy I/O, memory, and computations. @@ -163,7 +163,7 @@ The technique can only optimize `IN` queries that satisfy the following requirem (the combination of the columns uniquely identifies one particular column in the table). WARNING: -This technique will not improve the performance of the `COUNT(*)` queries. +This technique does not improve the performance of the `COUNT(*)` queries. ## The `InOperatorOptimization` module @@ -183,7 +183,7 @@ in `Gitlab::Pagination::Keyset::InOperatorOptimization`. ### Basic usage of `QueryBuilder` -To illustrate a basic usage, we will build a query that +To illustrate a basic usage, we build a query that fetches 20 issues with the oldest `created_at` from the group `gitlab-org`. The following ActiveRecord query would produce a query similar to @@ -226,10 +226,10 @@ Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new( the order by column expressions is available for locating the record. In this example, the yielded values are `created_at` and `id` SQL expressions. Finding a record is very fast via the primary key, so we don't use the `created_at` value. Providing the `finder_query` lambda is optional. - If it's not given, the IN operator optimization will only make the ORDER BY columns available to + If it's not given, the `IN` operator optimization only makes the `ORDER BY` columns available to the end-user and not the full database row. - If it's not given, the IN operator optimization will only make the ORDER BY columns available to + If it's not given, the `IN` operator optimization only makes the `ORDER BY` columns available to the end-user and not the full database row. The following database index on the `issues` table must be present @@ -416,7 +416,7 @@ scope = Issue .limit(20) ``` -To construct the array scope, we'll need to take the Cartesian product of the `project_id IN` and +To construct the array scope, we need to take the Cartesian product of the `project_id IN` and the `issue_type IN` queries. `issue_type` is an ActiveRecord enum, so we need to construct the following table: @@ -589,7 +589,7 @@ LIMIT 20 NOTE: To make the query efficient, the following columns need to be covered with an index: `project_id`, `issue_type`, `created_at`, and `id`. -#### Using calculated ORDER BY expression +#### Using calculated `ORDER BY` expression The following example orders epic records by the duration between the creation time and closed time. It is calculated with the following formula: @@ -766,7 +766,7 @@ using the generalized `IN` optimization technique. ### Array CTE -As the first step, we use a common table expression (CTE) for collecting the `projects.id` values. +As the first step, we use a Common Table Expression (CTE) for collecting the `projects.id` values. This is done by wrapping the incoming `array_scope` ActiveRecord relation parameter with a CTE. ```sql @@ -792,7 +792,7 @@ This query produces the following result set with only one column (`projects.id` ### Array mapping For each project (that is, each record storing a project ID in `array_cte`), -we will fetch the cursor value identifying the first issue respecting the `ORDER BY` clause. +we fetch the cursor value identifying the first issue respecting the `ORDER BY` clause. As an example, let's pick the first record `ID=9` from `array_cte`. The following query should fetch the cursor value `(created_at, id)` identifying @@ -805,7 +805,7 @@ ORDER BY "issues"."created_at" ASC, "issues"."id" ASC LIMIT 1; ``` -We will use `LATERAL JOIN` to loop over the records in the `array_cte` and find the +We use `LATERAL JOIN` to loop over the records in the `array_cte` and find the cursor value for each project. The query would be built using the `array_mapping_scope` lambda function. @@ -854,11 +854,11 @@ The table shows the cursor values (`created_at, id`) of the first record for eac respecting the `ORDER BY` clause. At this point, we have the initial data. To start collecting the actual records from the database, -we'll use a recursive CTE query where each recursion locates one row until +we use a recursive CTE query where each recursion locates one row until the `LIMIT` is reached or no more data can be found. -Here's an outline of the steps we will take in the recursive CTE query -(expressing the steps in SQL is non-trivial but will be explained next): +Here's an outline of the steps we take in the recursive CTE query +(expressing the steps in SQL is non-trivial but is explained next): 1. Sort the initial resultset according to the `ORDER BY` clause. 1. Pick the top cursor to fetch the record, this is our first record. In the example, @@ -877,7 +877,7 @@ this cursor would be (`2020-01-05`, `3`) for `project_id=9`. ### Initializing the recursive CTE query -For the initial recursive query, we'll need to produce exactly one row, we call this the +For the initial recursive query, we need to produce exactly one row, we call this the initializer query (`initializer_query`). Use `ARRAY_AGG` function to compact the initial result set into a single row @@ -994,7 +994,7 @@ After this, the recursion starts again by finding the next lowest cursor value. ### Finalizing the query -For producing the final `issues` rows, we're going to wrap the query with another `SELECT` statement: +For producing the final `issues` rows, we wrap the query with another `SELECT` statement: ```sql SELECT "issues".* @@ -1031,17 +1031,17 @@ Optimized `IN` query: | issue lookup query | 519 | 20 | 10 000 | The group and project queries are not using sorting, the necessary columns are read from database -indexes. These values are accessed frequently so it's very likely that most of the data will be +indexes. These values are accessed frequently so it's very likely that most of the data is in the PostgreSQL's buffer cache. -The optimized `IN` query will read maximum 519 entries (cursor values) from the index: +The optimized `IN` query reads maximum 519 entries (cursor values) from the index: - 500 index-only scans for populating the arrays for each project. The cursor values of the first -record will be here. +record is here. - Maximum 19 additional index-only scans for the consecutive records. -The optimized `IN` query will sort the array (cursor values per project array) 20 times, which -means we'll sort 20 x 500 rows. However, this might be a less memory-intensive task than +The optimized `IN` query sorts the array (cursor values per project array) 20 times, which +means we sort 20 x 500 rows. However, this might be a less memory-intensive task than sorting 10 000 rows at once. Performance comparison for the `gitlab-org` group: @@ -1053,5 +1053,5 @@ Performance comparison for the `gitlab-org` group: NOTE: Before taking measurements, the group lookup query was executed separately in order to make -the group data available in the buffer cache. Since it's a frequently called query, it's going to -hit many shared buffers during the query execution in the production environment. +the group data available in the buffer cache. Since it's a frequently called query, it +hits many shared buffers during the query execution in the production environment. diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 0363d13ed4c..b427f54ff3c 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/keyset_pagination.md b/doc/development/database/keyset_pagination.md index 88928feb927..4aec64b8cce 100644 --- a/doc/development/database/keyset_pagination.md +++ b/doc/development/database/keyset_pagination.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -178,7 +178,7 @@ To make keyset pagination work, we must configure custom order objects, to do so collect information about the order columns: - `relative_position` can have duplicated values because no unique index is present. -- `relative_position` can have null values because we don't have a not null constraint on the column. For this, we must determine where we see NULL values, at the beginning of the result set, or the end (`NULLS LAST`). +- `relative_position` can have null values because we don't have a not null constraint on the column. For this, we must determine where we see `NULL` values, at the beginning of the result set, or the end (`NULLS LAST`). - Keyset pagination requires distinct order columns, so we must add the primary key (`id`) to make the order distinct. - Jumping to the last page and paginating backwards actually reverses the `ORDER BY` clause. For this, we must provide the reversed `ORDER BY` clause. diff --git a/doc/development/database/layout_and_access_patterns.md b/doc/development/database/layout_and_access_patterns.md index a3e2fefb2a3..99a50b503aa 100644 --- a/doc/development/database/layout_and_access_patterns.md +++ b/doc/development/database/layout_and_access_patterns.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/loose_foreign_keys.md b/doc/development/database/loose_foreign_keys.md index 3db24793f1b..dec51d484fd 100644 --- a/doc/development/database/loose_foreign_keys.md +++ b/doc/development/database/loose_foreign_keys.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -255,7 +255,7 @@ When the loose foreign key definition is no longer needed (parent table is remov we need to remove the definition from the YAML file and ensure that we don't leave pending deleted records in the database. -1. Remove the loose foreign key definition from the config (`config/gitlab_loose_foreign_keys.yml`). +1. Remove the loose foreign key definition from the configuration (`config/gitlab_loose_foreign_keys.yml`). 1. Remove the deletion tracking trigger from the parent table (if the parent table is still there). 1. Remove leftover deleted records from the `loose_foreign_keys_deleted_records` table. @@ -429,7 +429,7 @@ ALTER TABLE ONLY vulnerability_occurrence_pipelines In this example we expect to delete all associated `vulnerability_occurrence_pipelines` records whenever we delete the `ci_pipelines` record associated with them. In this case you might end up with some vulnerability page in GitLab which shows an occurrence -of a vulnerability. However, when you try to click a link to the pipeline, you get +of a vulnerability. However, when you try to select a link to the pipeline, you get a 404, because the pipeline is deleted. Then, when you navigate back you might find the occurrence has disappeared too. @@ -515,13 +515,13 @@ referenced child tables. ### Database structure The feature relies on triggers installed on the parent tables. When a parent record is deleted, -the trigger will automatically insert a new record into the `loose_foreign_keys_deleted_records` +the trigger automatically inserts a new record into the `loose_foreign_keys_deleted_records` database table. -The inserted record will store the following information about the deleted record: +The inserted record stores the following information about the deleted record: - `fully_qualified_table_name`: name of the database table where the record was located. -- `primary_key_value`: the ID of the record, the value will be present in the child tables as +- `primary_key_value`: the ID of the record, the value is present in the child tables as the foreign key value. At the moment, composite primary keys are not supported, the parent table must have an `id` column. - `status`: defaults to pending, represents the status of the cleanup process. @@ -532,7 +532,7 @@ several runs. #### Database decomposition -The `loose_foreign_keys_deleted_records` table will exist on both database servers (Ci and Main) +The `loose_foreign_keys_deleted_records` table exists on both database servers (`ci` and `main`) after the [database decomposition](https://gitlab.com/groups/gitlab-org/-/epics/6168). The worker ill determine which parent tables belong to which database by reading the `lib/gitlab/database/gitlab_schemas.yml` YAML file. @@ -547,10 +547,10 @@ Example: - `ci_builds` - `ci_pipelines` -When the worker is invoked for the Ci database, the worker will load deleted records only from the +When the worker is invoked for the `ci` database, the worker loads deleted records only from the `ci_builds` and `ci_pipelines` tables. During the cleanup process, `DELETE` and `UPDATE` queries -will mostly run on tables located in the Main database. In this example, one `UPDATE` query will -nullify the `merge_requests.head_pipeline_id` column. +mostly run on tables located in the Main database. In this example, one `UPDATE` query +nullifies the `merge_requests.head_pipeline_id` column. #### Database partitioning @@ -561,7 +561,7 @@ strategy was considered for the feature but due to the large data volume we deci new strategy. A deleted record is considered fully processed when all its direct children records have been -cleaned up. When this happens, the loose foreign key worker will update the `status` column of +cleaned up. When this happens, the loose foreign key worker updates the `status` column of the deleted record. After this step, the record is no longer needed. The sliding partitioning strategy provides an efficient way of cleaning up old, unused data by @@ -591,7 +591,7 @@ Partitions: gitlab_partitions_dynamic.loose_foreign_keys_deleted_records_84 FOR ``` The `partition` column controls the insert direction, the `partition` value determines which -partition will get the deleted rows inserted via the trigger. Notice that the default value of +partition gets the deleted rows inserted via the trigger. Notice that the default value of the `partition` table matches with the value of the list partition (84). In `INSERT` query within the trigger the value of the `partition` is omitted, the trigger always relies on the default value of the column. @@ -607,20 +607,20 @@ SELECT TG_TABLE_SCHEMA || '.' || TG_TABLE_NAME, old_table.id FROM old_table; The partition "sliding" process is controlled by two, regularly executed callbacks. These callbacks are defined within the `LooseForeignKeys::DeletedRecord` model. -The `next_partition_if` callback controls when to create a new partition. A new partition will -be created when the current partition has at least one record older than 24 hours. A new partition +The `next_partition_if` callback controls when to create a new partition. A new partition is +created when the current partition has at least one record older than 24 hours. A new partition is added by the [`PartitionManager`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/partitioning/partition_manager.rb) using the following steps: 1. Create a new partition, where the `VALUE` for the partition is `CURRENT_PARTITION + 1`. 1. Update the default value of the `partition` column to `CURRENT_PARTITION + 1`. -With these steps, new `INSERT`-s via the triggers will end up in the new partition. At this point, +With these steps, all new `INSERT` queries via the triggers end up in the new partition. At this point, the database table has two partitions. The `detach_partition_if` callback determines if the old partitions can be detached from the table. A partition is detachable if there are no pending (unprocessed) records in the partition -(`status = 1`). The detached partitions will be available for some time, you can see the list +(`status = 1`). The detached partitions are available for some time, you can see the list detached partitions in the `detached_partitions` table: ```sql @@ -663,7 +663,7 @@ WHERE ("merge_requests"."id") IN These queries are batched, which means that in many cases, several invocations are needed to clean up all associated child records. -The batching is implemented with loops, the processing will stop when all associated child records +The batching is implemented with loops, the processing stops when all associated child records are cleaned up or the limit is reached. ```ruby @@ -682,14 +682,14 @@ end The loop-based batch processing is preferred over `EachBatch` for the following reasons: -- The records in the batch are modified, so the next batch will contain different records. +- The records in the batch are modified, so the next batch contains different records. - There is always an index on the foreign key column however, the column is usually not unique. `EachBatch` requires a unique column for the iteration. - The record order doesn't matter for the cleanup. -Notice that we have two loops. The initial loop will process records with the `SKIP LOCKED` clause. -The query will skip rows that are locked by other application processes. This will ensure that the -cleanup worker will less likely to become blocked. The second loop will execute the database +Notice that we have two loops. The initial loop processes records with the `SKIP LOCKED` clause. +The query skips rows that are locked by other application processes. This ensures that the +cleanup worker is less likely to become blocked. The second loop executes the database queries without `SKIP LOCKED` to ensure that all records have been processed. #### Processing limits @@ -709,19 +709,19 @@ To mitigate these issues, several limits are applied when the worker runs. The limit rules are implemented in the `LooseForeignKeys::ModificationTracker` class. When one of the limits (record modification count, time limit) is reached the processing is stopped -immediately. After some time, the next scheduled worker will continue the cleanup process. +immediately. After some time, the next scheduled worker continues the cleanup process. #### Performance characteristics -The database trigger on the parent tables will **decrease** the record deletion speed. Each -statement that removes rows from the parent table will invoke the trigger to insert records +The database trigger on the parent tables **decreases** the record deletion speed. Each +statement that removes rows from the parent table invokes the trigger to insert records into the `loose_foreign_keys_deleted_records` table. The queries within the cleanup worker are fairly efficient index scans, with limits in place they're unlikely to affect other parts of the application. The database queries are not running in transaction, when an error happens for example a statement -timeout or a worker crash, the next job will continue the processing. +timeout or a worker crash, the next job continues the processing. ## Troubleshooting @@ -730,13 +730,13 @@ timeout or a worker crash, the next job will continue the processing. There can be cases where the workers need to process an unusually large amount of data. This can happen under normal usage, for example when a large project or group is deleted. In this scenario, there can be several million rows to be deleted or nullified. Due to the limits enforced by the -worker, processing this data will take some time. +worker, processing this data takes some time. When cleaning up "heavy-hitters", the feature ensures fair processing by rescheduling larger batches for later. This gives time for other deleted records to be processed. For example, a project with millions of `ci_builds` records is deleted. The `ci_builds` records -will be deleted by the loose foreign keys feature. +is deleted by the loose foreign keys feature. 1. The cleanup worker is scheduled and picks up a batch of deleted `projects` records. The large project is part of the batch. @@ -746,7 +746,7 @@ project is part of the batch. 1. Go to step 1. The next cleanup worker continues the cleanup. 1. When the `cleanup_attempts` reaches 3, the batch is re-scheduled 10 minutes later by updating the `consume_after` column. -1. The next cleanup worker will process a different batch. +1. The next cleanup worker processes a different batch. We have Prometheus metrics in place to monitor the deleted record cleanup: @@ -812,7 +812,7 @@ runtime. LooseForeignKeys::CleanupWorker.new.perform ``` -When the cleanup is done, the older partitions will be automatically detached by the +When the cleanup is done, the older partitions are automatically detached by the `PartitionManager`. ### PartitionManager bug diff --git a/doc/development/database/maintenance_operations.md b/doc/development/database/maintenance_operations.md index 9e7a35531ca..85df185c024 100644 --- a/doc/development/database/maintenance_operations.md +++ b/doc/development/database/maintenance_operations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/migrations_for_multiple_databases.md b/doc/development/database/migrations_for_multiple_databases.md index ce326a6ce4a..df9607f5672 100644 --- a/doc/development/database/migrations_for_multiple_databases.md +++ b/doc/development/database/migrations_for_multiple_databases.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -13,11 +13,6 @@ for [the decomposed GitLab application using multiple databases](https://gitlab. Learn more about general multiple databases support in a [separate document](multiple_databases.md). -WARNING: -If you experience any issues using `Gitlab::Database::Migration[2.0]`, -you can temporarily revert back to the previous behavior by changing the version to `Gitlab::Database::Migration[1.0]`. -Please report any issues with `Gitlab::Database::Migration[2.0]` in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/358430). - The design for multiple databases (except for the Geo database) assumes that all decomposed databases have **the same structure** (for example, schema), but **the data is different** in each database. This means that some tables do not contain data on each database. @@ -78,6 +73,30 @@ class AddUserIdAndStateIndexToMergeRequestReviewers < Gitlab::Database::Migratio end ``` +#### Example: Add a new table to store in a single database + +1. Define the [GitLab Schema](multiple_databases.md#gitlab-schema) of the table in [`lib/gitlab/database/gitlab_schemas.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/gitlab_schemas.yml): + + ```yaml + ssh_signatures: :gitlab_main + ``` + +1. Create the table in a schema migration: + + ```ruby + class CreateSshSignatures < Gitlab::Database::Migration[2.0] + def change + create_table :ssh_signatures do |t| + t.timestamps_with_timezone null: false + t.bigint :project_id, null: false, index: true + t.bigint :key_id, null: false, index: true + t.integer :verification_status, default: 0, null: false, limit: 2 + t.binary :commit_sha, null: false, index: { unique: true } + end + end + end + ``` + ### Data Manipulation Language (DML) The DML migrations are all migrations that: @@ -241,7 +260,7 @@ the `database_tasks: false` set. `gitlab:db:validate_config` always runs before ## Validation -Validation in a nutshell uses [pg_query](https://github.com/pganalyze/pg_query) to analyze +Validation in a nutshell uses [`pg_query`](https://github.com/pganalyze/pg_query) to analyze each query and classify tables with information from [`gitlab_schema.yml`](multiple_databases.md#gitlab-schema). The migration is skipped if the specified `gitlab_schema` is outside of a list of schemas managed by a given database connection (`Gitlab::Database::gitlab_schemas_for_connection`). @@ -408,7 +427,7 @@ updating all `ci_pipelines`, you would set As with all DML migrations, you cannot query another database outside of `restrict_gitlab_migration` or `gitlab_shared`. If you need to query another database, -you'll likely need to separate these into two migrations somehow. +separate the migrations. Because the actual migration logic (not the queueing step) for background migrations runs in a Sidekiq worker, the logic can perform DML queries on diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index c622d4f50ff..7badd7f76fa 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Sharding info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -23,7 +23,8 @@ Each table of GitLab needs to have a `gitlab_schema` assigned: - `gitlab_main`: describes all tables that are being stored in the `main:` database (for example, like `projects`, `users`). - `gitlab_ci`: describes all CI tables that are being stored in the `ci:` database (for example, `ci_pipelines`, `ci_builds`). -- `gitlab_shared`: describe all application tables that contain data across all decomposed databases (for example, `loose_foreign_keys_deleted_records`). +- `gitlab_shared`: describe all application tables that contain data across all decomposed databases (for example, `loose_foreign_keys_deleted_records`) for models that inherit from `Gitlab::Database::SharedModel`. +- `gitlab_internal`: describe all internal tables of Rails and PostgreSQL (for example, `ar_internal_metadata`, `schema_migrations`, `pg_*`). - `...`: more schemas to be introduced with additional decomposed databases The usage of schema enforces the base class to be used: @@ -44,10 +45,8 @@ This is used as a primary source of classification for: ### The special purpose of `gitlab_shared` -`gitlab_shared` is a special case describing tables or views that by design contain data across -all decomposed databases. This does describe application-defined tables (like `loose_foreign_keys_deleted_records`), -Rails-defined tables (like `schema_migrations` or `ar_internal_metadata` as well as internal PostgreSQL tables -(for example, `pg_attribute`). +`gitlab_shared` is a special case that describes tables or views that, by design, contain data across +all decomposed databases. This classification describes application-defined tables (like `loose_foreign_keys_deleted_records`). **Be careful** to use `gitlab_shared` as it requires special handling while accessing data. Since `gitlab_shared` shares not only structure but also data, the application needs to be written in a way @@ -62,6 +61,11 @@ end As such, migrations modifying data of `gitlab_shared` tables are expected to run across all decomposed databases. +### The special purpose of `gitlab_internal` + +`gitlab_internal` describes Rails-defined tables (like `schema_migrations` or `ar_internal_metadata`), as well as internal PostgreSQL tables (for example, `pg_attribute`). Its primary purpose is to [support other databases](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85842#note_943453682), like Geo, that +might be missing some of those application-defined `gitlab_shared` tables (like `loose_foreign_keys_deleted_records`), but are valid Rails databases. + ## Migrations Read [Migrations for Multiple Databases](migrations_for_multiple_databases.md). @@ -597,3 +601,21 @@ way to replace cascading deletes so we don't end up with orphaned data or records that point to nowhere, which might lead to bugs. As such we created ["loose foreign keys"](loose_foreign_keys.md) which is an asynchronous process of cleaning up orphaned records. + +## Locking writes on the tables that don't belong to the database schemas + +When the CI database is promoted and the two databases are fully split, +as an extra safeguard against creating a split brain situation, +run the Rake task `gitlab:db:lock_writes`. This command locks writes on: + +- The `gitlab_main` tables on the CI Database. +- The `gitlab_ci` tables on the Main Database. + +This Rake task adds triggers to all the tables, to prevent any +`INSERT`, `UPDATE`, `DELETE`, or `TRUNCATE` statements from running +against the tables that need to be locked. + +If this task was run against a GitLab setup that uses only a single database +for both `gitlab_main` and `gitlab_ci` tables, then no tables will be locked. + +To undo the operation, run the opposite Rake task: `gitlab:db:unlock_writes`. diff --git a/doc/development/database/not_null_constraints.md b/doc/development/database/not_null_constraints.md index af7d569e282..3962307f80d 100644 --- a/doc/development/database/not_null_constraints.md +++ b/doc/development/database/not_null_constraints.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -99,8 +99,8 @@ such records, so we would follow the same process either way. We first add the `NOT NULL` constraint with a `NOT VALID` parameter, which enforces consistency when new records are inserted or current records are updated. -In the example above, the existing epics with a `NULL` description will not be affected and you'll -still be able to update records in the `epics` table. However, when you try to update or insert +In the example above, the existing epics with a `NULL` description are not affected and you are +still able to update records in the `epics` table. However, when you try to update or insert an epic without providing a description, the constraint causes a database error. Adding or removing a `NOT NULL` clause requires that any application changes are deployed _first_. @@ -129,7 +129,7 @@ end #### Data migration to fix existing records (current release) The approach here depends on the data volume and the cleanup strategy. The number of records that -must be fixed on GitLab.com is a nice indicator that will help us decide whether to use a +must be fixed on GitLab.com is a nice indicator that helps us decide whether to use a post-deployment migration or a background data migration: - If the data volume is less than `1000` records, then the data migration can be executed within the post-migration. @@ -138,7 +138,7 @@ post-deployment migration or a background data migration: When unsure about which option to use, please contact the Database team for advice. Back to our example, the epics table is not considerably large nor frequently accessed, -so we are going to add a post-deployment migration for the 13.0 milestone (current), +so we add a post-deployment migration for the 13.0 milestone (current), `db/post_migrate/20200501000002_cleanup_epics_with_null_description.rb`: ```ruby @@ -173,7 +173,7 @@ end #### Validate the `NOT NULL` constraint (next release) -Validating the `NOT NULL` constraint will scan the whole table and make sure that each record is correct. +Validating the `NOT NULL` constraint scans the whole table and make sure that each record is correct. Still in our example, for the 13.1 milestone (next), we run the `validate_not_null_constraint` migration helper in a final post-deployment migration, @@ -196,11 +196,11 @@ end ## `NOT NULL` constraints on large tables If you have to clean up a nullable column for a [high-traffic table](../migration_style_guide.md#high-traffic-tables) -(for example, the `artifacts` in `ci_builds`), your background migration will go on for a while and -it will need an additional [background migration cleaning up](background_migrations.md#cleaning-up) +(for example, the `artifacts` in `ci_builds`), your background migration goes on for a while and +it needs an additional [background migration cleaning up](background_migrations.md#cleaning-up) in the release after adding the data migration. -In that rare case you will need 3 releases end-to-end: +In that rare case you need 3 releases end-to-end: 1. Release `N.M` - Add the `NOT NULL` constraint and the background-migration to fix the existing records. 1. Release `N.M+1` - Cleanup the background migration. diff --git a/doc/development/database/pagination_guidelines.md b/doc/development/database/pagination_guidelines.md index 08840124535..1641708ce01 100644 --- a/doc/development/database/pagination_guidelines.md +++ b/doc/development/database/pagination_guidelines.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -28,9 +28,9 @@ We have two options for rendering the content: Rendering long lists can significantly affect both the frontend and backend performance: -- The database will need to read a lot of data from the disk. -- The result of the query (records) will eventually be transformed to Ruby objects which increases memory allocation. -- Large responses will take more time to send over the wire, to the user's browser. +- The database reads a lot of data from the disk. +- The result of the query (records) is eventually transformed to Ruby objects which increases memory allocation. +- Large responses take more time to send over the wire, to the user's browser. - Rendering long lists might freeze the browser (bad user experience). With pagination, the data is split into equal pieces (pages). On the first visit, the user receives only a limited number of items (page size). The user can see more items by paginating forward which results in a new HTTP request and a new database query. @@ -127,17 +127,17 @@ We can produce the same query in Rails: Issue.where(project_id: 1).page(1).per(20) ``` -The SQL query will return a maximum of 20 rows from the database. However, it doesn't mean that the database will only read 20 rows from the disk to produce the result. +The SQL query returns a maximum of 20 rows from the database. However, it doesn't mean that the database only reads 20 rows from the disk to produce the result. -This is what will happen: +This is what happens: -1. The database will try to plan the execution in the most efficient way possible based on the table statistics and the available indexes. +1. The database tries to plan the execution in the most efficient way possible based on the table statistics and the available indexes. 1. The planner knows that we have an index covering the `project_id` column. -1. The database will read all rows using the index on `project_id`. -1. The rows at this point are not sorted, so the database will need to sort the rows. +1. The database reads all rows using the index on `project_id`. +1. The rows at this point are not sorted, so the database sorts the rows. 1. The database returns the first 20 rows. -In case the project has 10_000 rows, the database will read 10_000 rows and sort them in memory (or on disk). This is not going to scale well in the long term. +In case the project has 10,000 rows, the database reads 10,000 rows and sorts them in memory (or on disk). This does not scale well in the long term. To fix this we need the following index: @@ -145,16 +145,16 @@ To fix this we need the following index: CREATE INDEX index_on_issues_project_id ON issues (project_id, id); ``` -By making the `id` column part of the index, the previous query will read maximum 20 rows. The query will perform well regardless of the number of issues within a project. So with this change, we've also improved the initial page load (when the user loads the issue page). +By making the `id` column part of the index, the previous query reads maximum 20 rows. The query performs well regardless of the number of issues within a project. So with this change, we've also improved the initial page load (when the user loads the issue page). NOTE: -Here we're leveraging the ordered property of the b-tree database index. Values in the index are sorted so reading 20 rows will not require further sorting. +Here we're leveraging the ordered property of the b-tree database index. Values in the index are sorted so reading 20 rows does not require further sorting. #### Limitations ##### `COUNT(*)` on a large dataset -Kaminari by default executes a count query to determine the number of pages for rendering the page links. Count queries can be quite expensive for a large table, in an unfortunate scenario the queries will simply time out. +Kaminari by default executes a count query to determine the number of pages for rendering the page links. Count queries can be quite expensive for a large table. In an unfortunate scenario the queries simply time out. To work around this, we can run Kaminari without invoking the count SQL query. @@ -162,11 +162,11 @@ To work around this, we can run Kaminari without invoking the count SQL query. Issue.where(project_id: 1).page(1).per(20).without_count ``` -In this case, the count query will not be executed and the pagination will no longer render the page numbers. We'll see only the next and previous links. +In this case, the count query is not executed and the pagination no longer renders the page numbers. We see only the next and previous links. ##### `OFFSET` on a large dataset -When we paginate over a large dataset, we might notice that the response time will get slower and slower. This is due to the `OFFSET` clause that seeks through the rows and skips N rows. +When we paginate over a large dataset, we might notice that the response time gets slower and slower. This is due to the `OFFSET` clause that seeks through the rows and skips N rows. From the user point of view, this might not be always noticeable. As the user paginates forward, the previous rows might be still in the buffer cache of the database. If the user shares the link with someone else and it's opened after a few minutes or hours, the response time might be significantly higher or it would even time out. @@ -214,7 +214,7 @@ Limit (cost=137878.89..137881.65 rows=20 width=1309) (actual time=5523.588..552 (8 rows) ``` -We can argue that a normal user will not be going to visit these pages, however, API users could easily navigate to very high page numbers (scraping, collecting data). +We can argue that a normal user does not visit these pages, however, API users could easily navigate to very high page numbers (scraping, collecting data). ### Keyset pagination @@ -279,7 +279,7 @@ eyJpZCI6Ijk0NzMzNTk0IiwidXBkYXRlZF9hdCI6IjIwMjEtMDQtMDkgMDg6NTA6MDUuODA1ODg0MDAw ``` NOTE: -Pagination parameters will be visible to the user, so we need to be careful about which columns we order by. +Pagination parameters are visible to the user, so be careful about which columns we order by. Keyset pagination can only provide the next, previous, first, and last pages. @@ -302,7 +302,7 @@ LIMIT 20 ##### Tooling -A generic keyset pagination library is available within the GitLab project which can most of the cases easily replace the existing, kaminari based pagination with significant performance improvements when dealing with large datasets. +A generic keyset pagination library is available within the GitLab project which can most of the cases easily replace the existing, Kaminari based pagination with significant performance improvements when dealing with large datasets. Example: diff --git a/doc/development/database/pagination_performance_guidelines.md b/doc/development/database/pagination_performance_guidelines.md index 90e4faf2de7..b5040e499e4 100644 --- a/doc/development/database/pagination_performance_guidelines.md +++ b/doc/development/database/pagination_performance_guidelines.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -55,13 +55,13 @@ LIMIT 20 OFFSET 0 ``` -With PostgreSQL version 11, the planner will first look up all issues matching the `project_id` filter and then join all `issue_metrics` rows. The ordering of rows will happen in memory. In case the joined relation is always present (1:1 relationship), the database will read `N * 2` rows where N is the number of rows matching the `project_id` filter. +With PostgreSQL version 11, the planner first looks up all issues matching the `project_id` filter and then join all `issue_metrics` rows. The ordering of rows happens in memory. In case the joined relation is always present (1:1 relationship), the database reads `N * 2` rows where N is the number of rows matching the `project_id` filter. For performance reasons, we should avoid mixing columns from different tables when specifying the `ORDER BY` clause. -In this particular case there is no simple way (like index creation) to improve the query. We might think that changing the `issues.id` column to `issue_metrics.issue_id` will help, however, this will likely make the query perform worse because it might force the database to process all rows in the `issue_metrics` table. +In this particular case there is no simple way (like index creation) to improve the query. We might think that changing the `issues.id` column to `issue_metrics.issue_id` helps, however, this likely makes the query perform worse because it might force the database to process all rows in the `issue_metrics` table. -One idea to address this problem is denormalization. Adding the `project_id` column to the `issue_metrics` table will make the filtering and sorting efficient: +One idea to address this problem is denormalization. Adding the `project_id` column to the `issue_metrics` table makes the filtering and sorting efficient: ```sql SELECT issues.* FROM issues @@ -73,7 +73,7 @@ OFFSET 0 ``` NOTE: -The query will require an index on `issue_metrics` table with the following column configuration: `(project_id, first_mentioned_in_commit_at DESC, issue_id DESC)`. +The query requires an index on `issue_metrics` table with the following column configuration: `(project_id, first_mentioned_in_commit_at DESC, issue_id DESC)`. ## Filtering @@ -81,7 +81,7 @@ The query will require an index on `issue_metrics` table with the following colu Filtering by a project is a very common use case since we have many features on the project level. Examples: merge requests, issues, boards, iterations. -These features will have a filter on `project_id` in their base query. Loading issues for a project: +These features have a filter on `project_id` in their base query. Loading issues for a project: ```ruby project = Project.find(5) @@ -108,9 +108,9 @@ This index fully covers the database query and the pagination. ### By group -Unfortunately, there is no efficient way to sort and paginate on the group level. The database query execution time will increase based on the number of records in the group. +Unfortunately, there is no efficient way to sort and paginate on the group level. The database query execution time increases based on the number of records in the group. -Things get worse when group level actually means group and its subgroups. To load the first page, the database needs to look up the group hierarchy, find all projects and then look up all issues. +Things get worse when group level actually means group and its subgroups. To load the first page, the database looks up the group hierarchy, finds all projects, and then looks up all issues. The main reason behind the inefficient queries on the group level is the way our database schema is designed; our core domain models are associated with a project, and projects are associated with groups. This doesn't mean that the database structure is bad, it's just in a well-normalized form that is not optimized for efficient group level queries. We might need to look into denormalization in the long term. @@ -184,7 +184,7 @@ LIMIT 20 OFFSET 0 ``` -Keep in mind that the index above will not support the following project level query: +The index above does not support the following project level query: ```sql SELECT "issues".* @@ -213,7 +213,7 @@ OFFSET 0 We might be tempted to add an index on `project_id`, `confidential`, and `iid` to improve the database query, however, in this case it's probably unnecessary. Based on the data distribution in the table, confidential issues are rare. Filtering them out does not make the database query significantly slower. The database might read a few extra rows, the performance difference might not even be visible to the end-user. -On the other hand, if we would implement a special filter where we only show confidential issues, we will surely need the index. Finding 20 confidential issues might require the database to scan hundreds of rows or in the worst case, all issues in the project. +On the other hand, if we implemented a special filter where we only show confidential issues, we need the index. Finding 20 confidential issues might require the database to scan hundreds of rows or, in the worst case, all issues in the project. NOTE: Be aware of the data distribution and the table access patterns (how features work) when introducing a new database index. Sampling production data might be necessary to make the right decision. @@ -253,7 +253,7 @@ Example database (oversimplified) execution plan: - `SELECT "issues".* FROM "issues" WHERE "issues"."project_id" = 5` 1. The database estimates the number of rows and the costs to run these queries. 1. The database executes the cheapest query first. -1. Using the query result, load the rows from the other table (from the other query) using the JOIN column and filter the rows further. +1. Using the query result, load the rows from the other table (from the other query) using the `JOIN` column and filter the rows further. In this particular example, the `issue_assignees` query would likely be executed first. @@ -276,17 +276,17 @@ Running the query in production for the GitLab project produces the following ex (13 rows) ``` -The query looks up the `assignees` first, filtered by the `user_id` (`user_id = 4156052`) and it finds 215 rows. Using that 215 rows, the database will look up the 215 associated issue rows by the primary key. Notice that the filter on the `project_id` column is not backed by an index. +The query looks up the `assignees` first, filtered by the `user_id` (`user_id = 4156052`) and it finds 215 rows. Using those 215 rows, the database looks up the 215 associated issue rows by the primary key. Notice that the filter on the `project_id` column is not backed by an index. -In most cases, we are lucky that the joined relation will not be going to return too many rows, therefore, we will end up with a relatively efficient database query that accesses low number of rows. As the database grows, these queries might start to behave differently. Let's say the number `issue_assignees` records for a particular user is very high (millions), then this join query will not perform well, and it will likely time out. +In most cases, we are lucky that the joined relation does not return too many rows, therefore, we end up with a relatively efficient database query that accesses a small number of rows. As the database grows, these queries might start to behave differently. Let's say the number `issue_assignees` records for a particular user is very high, in the millions. This join query does not perform well, and it likely times out. -A similar problem could be a double join, where the filter exists in the 2nd JOIN query. Example: `Issue -> LabelLink -> Label(name=bug)`. +A similar problem could be a double join, where the filter exists in the 2nd `JOIN` query. Example: `Issue -> LabelLink -> Label(name=bug)`. There is no easy way to fix these problems. Denormalization of data could help significantly, however, it has also negative effects (data duplication and keeping the data up to date). Ideas for improving the `issue_assignees` filter: -- Add `project_id` column to the `issue_assignees` table so when JOIN-ing, the extra `project_id` filter will further filter the rows. The sorting will likely happen in memory: +- Add `project_id` column to the `issue_assignees` table so when performing the `JOIN`, the extra `project_id` filter further filters the rows. The sorting likely happens in memory: ```sql SELECT "issues".* diff --git a/doc/development/database/post_deployment_migrations.md b/doc/development/database/post_deployment_migrations.md index 799eefdb875..a49c77ca047 100644 --- a/doc/development/database/post_deployment_migrations.md +++ b/doc/development/database/post_deployment_migrations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/rename_database_tables.md b/doc/development/database/rename_database_tables.md index 7a76c028042..cbcbd507204 100644 --- a/doc/development/database/rename_database_tables.md +++ b/doc/development/database/rename_database_tables.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -135,4 +135,4 @@ database, ActiveRecord fetches the column information again. At this time, our p marked table (`TABLES_TO_BE_RENAMED`) instructs ActiveRecord to use the new database table name when fetching the database table information. -The new version of the application will use the new database table. +The new version of the application uses the new database table. diff --git a/doc/development/database/setting_multiple_values.md b/doc/development/database/setting_multiple_values.md index 0f23aae9f79..cba15a73430 100644 --- a/doc/development/database/setting_multiple_values.md +++ b/doc/development/database/setting_multiple_values.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index 7aa529e1518..73e023f8d45 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30453) in GitLab 13.0. -When adding new columns that will be used to store strings or other textual information: +When adding new columns to store strings or other textual information: 1. We always use the `text` data type instead of the `string` data type. 1. `text` columns should always have a limit set, either by using the `create_table` with @@ -142,8 +142,8 @@ instance of GitLab could have such records, so we would follow the same process We first add the limit as a `NOT VALID` check constraint to the table, which enforces consistency when new records are inserted or current records are updated. -In the example above, the existing issues with more than 1024 characters in their title will not be -affected and you'll be still able to update records in the `issues` table. However, when you'd try +In the example above, the existing issues with more than 1024 characters in their title are not +affected, and you are still able to update records in the `issues` table. However, when you'd try to update the `title_html` with a title that has more than 1024 characters, the constraint causes a database error. @@ -182,7 +182,7 @@ end #### Data migration to fix existing records (current release) The approach here depends on the data volume and the cleanup strategy. The number of records that must -be fixed on GitLab.com is a nice indicator that will help us decide whether to use a post-deployment +be fixed on GitLab.com is a nice indicator that helps us decide whether to use a post-deployment migration or a background data migration: - If the data volume is less than `1,000` records, then the data migration can be executed within the post-migration. @@ -233,7 +233,7 @@ You can find more information on the guide about [background migrations](backgro #### Validate the text limit (next release) -Validating the text limit will scan the whole table and make sure that each record is correct. +Validating the text limit scans the whole table, and makes sure that each record is correct. Still in our example, for the 13.1 milestone (next), we run the `validate_text_limit` migration helper in a final post-deployment migration, @@ -276,11 +276,11 @@ end ## Text limit constraints on large tables If you have to clean up a text column for a really [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) -(for example, the `artifacts` in `ci_builds`), your background migration will go on for a while and -it will need an additional [background migration cleaning up](background_migrations.md#cleaning-up) +(for example, the `artifacts` in `ci_builds`), your background migration goes on for a while and +it needs an additional [background migration cleaning up](background_migrations.md#cleaning-up) in the release after adding the data migration. -In that rare case you will need 3 releases end-to-end: +In that rare case you need 3 releases end-to-end: 1. Release `N.M` - Add the text limit and the background migration to fix the existing records. 1. Release `N.M+1` - Cleanup the background migration. diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md index 34cb73978bc..582c988bef9 100644 --- a/doc/development/database/table_partitioning.md +++ b/doc/development/database/table_partitioning.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -36,23 +36,23 @@ before attempting to leverage this feature. While partitioning can be very useful when properly applied, it's imperative to identify if the data and workload of a table naturally fit a -partitioning scheme. There are a few details you'll have to understand -in order to decide if partitioning is a good fit for your particular +partitioning scheme. There are a few details you have to understand +to decide if partitioning is a good fit for your particular problem. First, a table is partitioned on a partition key, which is a column or -set of columns which determine how the data will be split across the +set of columns which determine how the data is split across the partitions. The partition key is used by the database when reading or -writing data, to decide which partitions need to be accessed. The +writing data, to decide which partitions must be accessed. The partition key should be a column that would be included in a `WHERE` clause on almost all queries accessing that table. -Second, it's necessary to understand the strategy the database will -use to split the data across the partitions. The scheme supported by the +Second, it's necessary to understand the strategy the database uses +to split the data across the partitions. The scheme supported by the GitLab migration helpers is date-range partitioning, where each partition in the table contains data for a single month. In this case, the partitioning -key would need to be a timestamp or date column. In order for this type of -partitioning to work well, most queries would need to access data within a +key must be a timestamp or date column. In order for this type of +partitioning to work well, most queries must access data in a certain date range. For a more concrete example, the `audit_events` table can be used, which @@ -73,7 +73,7 @@ CREATE TABLE audit_events ( created_at timestamptz NOT NULL); ``` -Now imagine typical queries in the UI would display the data within a +Now imagine typical queries in the UI would display the data in a certain date range, like a single week: ```sql @@ -117,7 +117,7 @@ partition key falls in the specified range. For example, the partition greater than or equal to `2020-01-01` and less than `2020-02-01`. Now, if we look at the previous example query again, the database can -use the `WHERE` to recognize that all matching rows will be in the +use the `WHERE` to recognize that all matching rows are in the `audit_events_202001` partition. Rather than searching all of the data in all of the partitions, it can search only the single month's worth of data in the appropriate partition. In a large table, this can @@ -136,11 +136,11 @@ LIMIT 100 In this example, the database can't prune any partitions from the search, because matching data could exist in any of them. As a result, it has to query each partition individually, and aggregate the rows into a single result -set. Since `author_id` would be indexed, the performance impact could +set. Because `author_id` would be indexed, the performance impact could likely be acceptable, but on more complex queries the overhead can be substantial. Partitioning should only be leveraged if the access patterns -of the data support the partitioning strategy, otherwise performance will -suffer. +of the data support the partitioning strategy, otherwise performance +suffers. ## Partitioning a table @@ -158,15 +158,15 @@ migration to copy data into the new table. Changes to the original table schema can be made in parallel with the partitioning migration, but they must take care to not break the underlying mechanism that makes the migration work. For example, if a column is added to the table that is being -partitioned, both the partitioned table and the trigger definition need to +partitioned, both the partitioned table and the trigger definition must be updated to match. ### Step 1: Creating the partitioned copy (Release N) The first step is to add a migration to create the partitioned copy of -the original table. This migration will also create the appropriate +the original table. This migration creates the appropriate partitions based on the data in the original table, and install a -trigger that will sync writes from the original table into the +trigger that syncs writes from the original table into the partitioned copy. An example migration of partitioning the `audit_events` table by its @@ -186,15 +186,15 @@ class PartitionAuditEvents < Gitlab::Database::Migration[1.0] end ``` -Once this has executed, any inserts, updates or deletes in the -original table will also be duplicated in the new table. For updates and -deletes, the operation will only have an effect if the corresponding row +After this has executed, any inserts, updates, or deletes in the +original table are also duplicated in the new table. For updates and +deletes, the operation only has an effect if the corresponding row exists in the partitioned table. ### Step 2: Backfill the partitioned copy (Release N) -The second step is to add a post-deployment migration that will schedule -the background jobs that will backfill existing data from the original table +The second step is to add a post-deployment migration that schedules +the background jobs that backfill existing data from the original table into the partitioned copy. Continuing the above example, the migration would look like: @@ -225,7 +225,7 @@ partitioning migration. The third step must occur at least one release after the release that includes the background migration. This gives time for the background migration to execute properly in self-managed installations. In this step, -add another post-deployment migration that will cleanup after the +add another post-deployment migration that cleans up after the background migration. This includes forcing any remaining jobs to execute, and copying data that may have been missed, due to dropped or failed jobs. @@ -248,12 +248,11 @@ end After this migration has completed, the original table and partitioned table should contain identical data. The trigger installed on the -original table guarantees that the data will remain in sync going -forward. +original table guarantees that the data remains in sync going forward. ### Step 4: Swap the partitioned and non-partitioned tables (Release N+1) -The final step of the migration will make the partitioned table ready +The final step of the migration makes the partitioned table ready for use by the application. This section will be updated when the migration helper is ready, for now development can be followed in the [Tracking Issue](https://gitlab.com/gitlab-org/gitlab/-/issues/241267). diff --git a/doc/development/database/transaction_guidelines.md b/doc/development/database/transaction_guidelines.md index 2806bd217db..d96d11f05a5 100644 --- a/doc/development/database/transaction_guidelines.md +++ b/doc/development/database/transaction_guidelines.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -15,8 +15,8 @@ For further reference, check PostgreSQL documentation about [transactions](https The [sharding group](https://about.gitlab.com/handbook/engineering/development/enablement/sharding/) plans to split the main GitLab database and move some of the database tables to other database servers. -We'll start decomposing the `ci_*`-related database tables first. To maintain the current application -development experience, we'll add tooling and static analyzers to the codebase to ensure correct +We start decomposing the `ci_*`-related database tables first. To maintain the current application +development experience, we add tooling and static analyzers to the codebase to ensure correct data access and data modification methods. By using the correct form for defining database transactions, we can save significant refactoring work in the future. @@ -60,7 +60,7 @@ end The database tries to acquire the `FOR UPDATE` lock for the referenced `issue` and `project` records. In our case, we have two competing transactions for these locks, -and only one of them will successfully acquire them. The other transaction will have +and only one of them successfully acquires them. The other transaction has to wait in the lock queue until the first transaction finishes. The execution of the second transaction is blocked at this point. @@ -139,5 +139,5 @@ end ``` The `ActiveRecord::Base` class uses a different database connection than the `Ci::Build` records. -The two statements in the transaction block will not be part of the transaction and will not be +The two statements in the transaction block are not part of the transaction and are rolled back in case something goes wrong. They act as 3rd part calls. diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 426d355bd82..5d46ade98bb 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This section is to help give some copy-pasta you can use as a reference when you run into some head-banging database problems. -An easy first step is to search for your error in Slack, or search for `GitLab <my error>` with Google. +A first step is to search for your error in Slack, or search for `GitLab <my error>` with Google. Available `RAILS_ENV`: @@ -57,7 +57,7 @@ bundle exec rake db:reset RAILS_ENV=test Access the database via one of these commands (they all get you to the same place) -```ruby +```shell gdk psql -d gitlabhq_development bundle exec rails dbconsole -e development bundle exec rails db -e development @@ -72,7 +72,7 @@ bundle exec rails db -e development ## Access the database with a GUI -Most GUIs (DataGrid, RubyMine, DBeaver) require a TCP connection to the database, but by default +Most GUIs (DataGrip, RubyMine, DBeaver) require a TCP connection to the database, but by default the database runs on a UNIX socket. To be able to access the database from these tools, some steps are needed: @@ -106,8 +106,8 @@ Use these instructions for exploring the GitLab database while developing with t 1. Install or open [Visual Studio Code](https://code.visualstudio.com/download). 1. Install the [PostgreSQL VSCode Extension](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres). -1. In Visual Studio Code click on the PostgreSQL Explorer button in the left toolbar. -1. In the top bar of the new window, click on the `+` to **Add Database Connection**, and follow the prompts to fill in the details: +1. In Visual Studio Code select **PostgreSQL Explorer** in the left toolbar. +1. In the top bar of the new window, select `+` to **Add Database Connection**, and follow the prompts to fill in the details: 1. **Hostname**: the path to the PostgreSQL folder in your GDK directory (for example `/dev/gitlab-development-kit/postgresql`). 1. **PostgreSQL user to authenticate as**: usually your local username, unless otherwise specified during PostgreSQL installation. 1. **Password of the PostgreSQL user**: the password you set when installing PostgreSQL. @@ -169,7 +169,7 @@ possible to migrate GitLab from every previous version. In some cases you may want to bypass this check. For example, if you were on a version of GitLab schema later than the `MIN_SCHEMA_VERSION`, and then rolled back the -to an older migration, from before. In this case, in order to migrate forward again, +to an older migration, from before. In this case, to migrate forward again, you should set the `SKIP_SCHEMA_VERSION_CHECK` environment variable. ```shell diff --git a/doc/development/database_query_comments.md b/doc/development/database_query_comments.md index e4133633a77..2798071bc06 100644 --- a/doc/development/database_query_comments.md +++ b/doc/development/database_query_comments.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -12,7 +12,7 @@ queries generated by ActiveRecord. It is very useful for tracing problematic queries back to the application source. -An engineer during an on-call incident will have the full context of a query +An engineer during an on-call incident has the full context of a query and its application source from the comments. ## Metadata information in comments @@ -24,7 +24,7 @@ Queries generated from **Rails** include the following metadata in comments: - `endpoint_id` - `line` -Queries generated from **Sidekiq** workers will include the following metadata +Queries generated from **Sidekiq** workers include the following metadata in comments: - `application` diff --git a/doc/development/database_review.md b/doc/development/database_review.md index fd0e2e17623..2b215190e6d 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -1,12 +1,12 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Database Review Guidelines -This page is specific to database reviews. Please refer to our +This page is specific to database reviews. Refer to our [code review guide](code_review.md) for broader advice and best practices for code review in general. @@ -25,7 +25,7 @@ A database review is required for: generally up to the author of a merge request to decide whether or not complex queries are being introduced and if they require a database review. -- Changes in Service Data metrics that use `count`, `distinct_count` and `estimate_batch_distinct_count`. +- Changes in Service Data metrics that use `count`, `distinct_count`, `estimate_batch_distinct_count` and `sum`. These metrics could have complex queries over large tables. See the [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) for implementation details. @@ -39,7 +39,7 @@ migration only. ### Required You must provide the following artifacts when you request a ~database review. -If your merge request description does not include these items, the review will be reassigned back to the author. +If your merge request description does not include these items, the review is reassigned back to the author. #### Migrations @@ -47,7 +47,7 @@ If new migrations are introduced, in the MR **you are required to provide**: - The output of both migrating (`db:migrate`) and rolling back (`db:rollback`) for all migrations. -Note that we have automated tooling for +We have automated tooling for [GitLab](https://gitlab.com/gitlab-org/gitlab) (provided by the [`db:check-migrations`](database/dbcheck-migrations-job.md) pipeline job) that provides this output for migrations on ~database merge requests. You do not need to provide this information manually @@ -88,7 +88,7 @@ A database **maintainer**'s role is to: database reviewer and the MR author. - Finally approve the MR and relabel the MR with ~"database::approved" - Merge the MR if no other approvals are pending or pass it on to - other maintainers as required (frontend, backend, docs). + other maintainers as required (frontend, backend, documentation). - If not merging, remove yourself as a reviewer. ### Distributing review workload @@ -96,7 +96,7 @@ A database **maintainer**'s role is to: Review workload is distributed using [reviewer roulette](code_review.md#reviewer-roulette) ([example](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25181#note_147551725)). The MR author should request a review from the suggested database -**reviewer**. When they give their sign-off, they will hand over to +**reviewer**. When they sign off, they hand over to the suggested database **maintainer**. If reviewer roulette didn't suggest a database reviewer & maintainer, @@ -106,7 +106,7 @@ make sure you have applied the ~database label and rerun the ### How to prepare the merge request for a database review -In order to make reviewing easier and therefore faster, please take +To make reviewing easier and therefore faster, take the following preparations into account. #### Preparation when adding migrations @@ -124,10 +124,10 @@ the following preparations into account. - When adding an index to a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slack channel and add the execution time to the MR description: - Execution time largely varies between `#database-lab` and GitLab.com, but an elevated execution time from `#database-lab` - can give a hint that the execution on GitLab.com will also be considerably high. + can give a hint that the execution on GitLab.com is also considerably high. - If the execution from `#database-lab` is longer than `1h`, the index should be moved to a [post-migration](database/post_deployment_migrations.md). Keep in mind that in this case you may need to split the migration and the application changes in separate releases to ensure the index - will be in place when the code that needs it will be deployed. + is in place when the code that needs it is deployed. - Manually trigger the [database testing](database/database_migration_pipeline.md) job (`db:gitlabcom-database-testing`) in the `test` stage. - This job runs migrations in a production-like environment (similar to `#database_lab`) and posts to the MR its findings (queries, runtime, size change). - Review migration runtimes and any warnings. @@ -139,10 +139,10 @@ of error that would result in corruption or loss of production data. Include in the MR description: -- If the migration itself is not reversible, details of how data changes could be reverted in the event of an incident. For example, in the case of a migration that deletes records (an operation that most of the times is not automatically revertable), how _could_ the deleted records be recovered. +- If the migration itself is not reversible, details of how data changes could be reverted in the event of an incident. For example, in the case of a migration that deletes records (an operation that most of the times is not automatically reversible), how _could_ the deleted records be recovered. - If the migration deletes data, apply the label `~data-deletion`. - Concise descriptions of possible user experience impact of an error; for example, "Issues would unexpectedly go missing from Epics". -- Relevant data from the [query plans](#query-plans) that indicate the query works as expected; such as the approximate number of records that will be modified/deleted. +- Relevant data from the [query plans](#query-plans) that indicate the query works as expected; such as the approximate number of records that are modified or deleted. #### Preparation when adding or modifying queries @@ -151,13 +151,13 @@ Include in the MR description: - Write the raw SQL in the MR description. Preferably formatted nicely with [pgFormatter](https://sqlformat.darold.net) or [paste.depesz.com](https://paste.depesz.com) and using regular quotes - <!-- vale off --> +<!-- vale gitlab.NonStandardQuotes = NO --> (for example, `"projects"."id"`) and avoiding smart quotes (for example, `“projects”.“id”`). - <!-- vale on --> +<!-- vale gitlab.NonStandardQuotes = YES --> - In case of queries generated dynamically by using parameters, there should be one raw SQL query for each variation. For example, a finder for issues that may take as a parameter an optional filter on projects, - should include both the version of the simple query over issues and the one that joins issues + should include both the version of the query over issues and the one that joins issues and projects and applies the filter. There are finders or other methods that can generate a very large amount of permutations. @@ -167,7 +167,7 @@ Include in the MR description: For example, if joins or a group by clause are optional, the versions without the group by clause and with less joins should be also included, while keeping the appropriate filters for the remaining tables. -- If a query is going to be always used with a limit and an offset, those should always be +- If a query is always used with a limit and an offset, those should always be included with the maximum allowed limit used and a non 0 offset. ##### Query Plans @@ -175,7 +175,7 @@ Include in the MR description: - The query plan for each raw SQL query included in the merge request along with the link to the query plan following each raw SQL snippet. - Provide a public link to the plan from either: - [postgres.ai](https://postgres.ai/): Follow the link in `#database-lab` and generate a shareable, public link - by clicking the **Share** button in the upper right corner. + by clicking **Share** in the upper right corner. - [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com): Paste both the plan and the query used in the form. - When providing query plans, make sure it hits enough data: - You can use a GitLab production replica to test your queries on a large scale, @@ -204,11 +204,11 @@ Include in the MR description: - Add foreign keys to any columns pointing to data in other tables, including [an index](migration_style_guide.md#adding-foreign-key-constraints). - Add indexes for fields that are used in statements such as `WHERE`, `ORDER BY`, `GROUP BY`, and `JOIN`s. - New tables and columns are not necessarily risky, but over time some access patterns are inherently - difficult to scale. To identify these risky patterns in advance, we need to document expectations for + difficult to scale. To identify these risky patterns in advance, we must document expectations for access and size. Include in the MR description answers to these questions: - What is the anticipated growth for the new table over the next 3 months, 6 months, 1 year? What assumptions are these based on? - How many reads and writes per hour would you expect this table to have in 3 months, 6 months, 1 year? Under what circumstances are rows updated? What assumptions are these based on? - - Based on the anticipated data volume and access patterns, does the new table pose an availability risk to GitLab.com or self-managed instances? Will the proposed design scale to support the needs of GitLab.com and self-managed customers? + - Based on the anticipated data volume and access patterns, does the new table pose an availability risk to GitLab.com or self-managed instances? Does the proposed design scale to support the needs of GitLab.com and self-managed customers? #### Preparation when removing columns, tables, indexes, or other structures @@ -235,7 +235,7 @@ Include in the MR description: - Check consistency with `db/structure.sql` and that migrations are [reversible](migration_style_guide.md#reversibility) - Check that the relevant version files under `db/schema_migrations` were added or removed. - Check queries timing (If any): In a single transaction, cumulative query time executed in a migration - needs to fit comfortably within `15s` - preferably much less than that - on GitLab.com. + needs to fit comfortably in `15s` - preferably much less than that - on GitLab.com. - For column removals, make sure the column has been [ignored in a previous release](database/avoiding_downtime_in_migrations.md#dropping-columns) - Check [background migrations](database/background_migrations.md): - Establish a time estimate for execution on GitLab.com. For historical purposes, @@ -266,7 +266,7 @@ Include in the MR description: - Query performance - Check for any overly complex queries and queries the author specifically points out for review (if any) - - If not present yet, ask the author to provide SQL queries and query plans + - If not present, ask the author to provide SQL queries and query plans (for example, by using [ChatOps](understanding_explain_plans.md#chatops) or direct database access) - For given queries, review parameters regarding data distribution diff --git a/doc/development/db_dump.md b/doc/development/db_dump.md index 0c63bf06e07..f2076cbc410 100644 --- a/doc/development/db_dump.md +++ b/doc/development/db_dump.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/deprecation_guidelines/img/deprecation_removal_process.png b/doc/development/deprecation_guidelines/img/deprecation_removal_process.png Binary files differnew file mode 100644 index 00000000000..99642ebbae0 --- /dev/null +++ b/doc/development/deprecation_guidelines/img/deprecation_removal_process.png diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md index cafc40ccc68..7fbe2261f4d 100644 --- a/doc/development/deprecation_guidelines/index.md +++ b/doc/development/deprecation_guidelines/index.md @@ -11,13 +11,31 @@ changes](../contributing/index.md#breaking-changes) to GitLab features. ## Terminology -It's important to understand the difference between **deprecation** and -**removal**: +**Deprecation**: -**Deprecation** is the process of flagging/marking/announcing that a feature is no longer fully supported and may be removed in a future version of GitLab. +- Feature not recommended for use. +- Development restricted to Priority 1 / Severity 1 bug fixes. +- Will be removed in a future major release. +- Begins after a deprecation announcement outlining an end-of-support date. +- Ends after the end-of-support date or removal date has passed. -**Removal** is the process of actually removing a feature that was previously -deprecated. +**End of Support**: + +- Feature usage strongly discouraged. +- No support or fixes provided. +- No longer tested internally. +- Will be removed in a future major release. +- Begins after an end-of-support date has passed. +- Ends after all relevant code has been removed. + +**Removal**: + +- Feature usage impossible. +- Happens in a major release in line with our + [semantic versioning policy](../../policy/maintenance.md). +- Begins after removal date has passed. + + ## When can a feature be deprecated? diff --git a/doc/development/developing_with_solargraph.md b/doc/development/developing_with_solargraph.md index 877fbad8ab2..d7e41187ace 100644 --- a/doc/development/developing_with_solargraph.md +++ b/doc/development/developing_with_solargraph.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Gemfile packages [Solargraph](https://github.com/castwide/solargraph) language server for additional IntelliSense and code formatting capabilities with editors that support it. -Example configuration for Solargraph can be found in [.solargraph.yml.example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.solargraph.yml.example) file. Copy the contents of this file to `.solargraph.yml` file for language server to pick this configuration up. Since `.solargraph.yml` configuration file is ignored by Git, it's possible to adjust configuration according to your needs. +Example configuration for Solargraph can be found in [`.solargraph.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.solargraph.yml.example) file. Copy the contents of this file to `.solargraph.yml` file for language server to pick this configuration up. Since `.solargraph.yml` configuration file is ignored by Git, it's possible to adjust configuration according to your needs. Refer to particular IDE plugin documentation on how to integrate it with Solargraph language server: diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index b4f347449cc..116071cdfd9 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -71,7 +71,7 @@ GITLAB_TRACING=opentracing://<driver>?<param_name>=<param_value>&<param_name_2>= In this example, we have the following hypothetical values: -- `driver`: the driver such a jaegar. +- `driver`: the driver such a Jaeger. - `param_name`, `param_value`: these are driver specific configuration values. Configuration parameters for Jaeger are documented [further on in this document](#2-configure-the-gitlab_tracing-environment-variable) they should be URL encoded. @@ -87,7 +87,7 @@ The easiest way to access tracing from a GDK environment is through the [performance-bar](../administration/monitoring/performance/performance_bar.md). This can be shown by typing `p` `b` in the browser window. -Once the performance bar is enabled, click on the **Trace** link in the performance bar to go to +Once the performance bar is enabled, select **Trace** in the performance bar to go to the Jaeger UI. The Jaeger search UI returns a query for the `Correlation-ID` of the current request. Normally, diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index c5ea1985fc7..89e54183e50 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -29,11 +29,11 @@ When the state of a flag changes (for example, disabled by default to enabled by Possible version history entries are: ```markdown -> - [Introduced](issue-link) in GitLab X.X [with a flag](../../administration/feature_flags.md) named <flag name>. Disabled by default. +> - [Introduced](issue-link) in GitLab X.X [with a flag](../../administration/feature_flags.md) named `flag_name`. Disabled by default. > - [Enabled on GitLab.com](issue-link) in GitLab X.X. > - [Enabled on GitLab.com](issue-link) in GitLab X.X. Available to GitLab.com administrators only. > - [Enabled on self-managed](issue-link) in GitLab X.X. -> - [Generally available](issue-link) in GitLab X.Y. [Feature flag <flag name>](issue-link) removed. +> - [Generally available](issue-link) in GitLab X.Y. [Feature flag `flag_name`](issue-link) removed. ``` You can combine entries if they happened in the same release: @@ -60,15 +60,15 @@ FLAG: | If the feature is... | Use this text | |--------------------------|---------------| -| Available | `On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Unavailable | `On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Available to some users | `On self-managed GitLab, by default this feature is available to a subset of users. To show or hide the feature for all, ask an administrator to [change the status of the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Available, per-group | `On self-managed GitLab, by default this feature is available. To hide the feature per group, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Unavailable, per-group | `On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Available, per-project | `On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Unavailable, per-project | `On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Available, per-user | `On self-managed GitLab, by default this feature is available. To hide the feature per user, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | -| Unavailable, per-user | `On self-managed GitLab, by default this feature is not available. To make it available per user, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` | +| Available | ``On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Unavailable | ``On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Available to some users | ``On self-managed GitLab, by default this feature is available to a subset of users. To show or hide the feature for all, ask an administrator to [change the status of the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Available, per-group | ``On self-managed GitLab, by default this feature is available. To hide the feature per group, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Unavailable, per-group | ``On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Available, per-project | ``On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Unavailable, per-project | ``On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Available, per-user | ``On self-managed GitLab, by default this feature is available. To hide the feature per user, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | +| Unavailable, per-user | ``On self-managed GitLab, by default this feature is not available. To make it available per user, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named `flag_name`.`` | ### GitLab.com availability information @@ -114,5 +114,5 @@ And, when the feature is done and fully available to all users: > - Introduced in GitLab 13.7 [with a flag](../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default. > - [Enabled on self-managed](https://gitlab.com/issue/etc) in GitLab 13.8. > - [Enabled on GitLab.com](https://gitlab.com/issue/etc) in GitLab 13.9. -> - [Generally available](issue-link) in GitLab 14.0. [Feature flag <flag name>](issue-link) removed. +> - [Generally available](issue-link) in GitLab 14.0. [Feature flag `forti_token_cloud`](issue-link) removed. ``` diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index c6afcdbddd0..ee439e93011 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -143,6 +143,71 @@ Nanoc layout), which is displayed at the top of the page if defined. The `type` metadata parameter is deprecated but still exists in documentation pages. You can safely remove the `type` metadata parameter and its values. +### Batch updates for TW metadata + +NOTE: +This task is an MVC, and requires significant manual preparation of the output. +While the task can be time consuming, it is still faster than doing the work +entirely manually. + +It's important to keep the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS) +file in the `gitlab` project up to date with the current Technical Writing team assignments. +This information is used in merge requests that contain documentation: + +- To populate the eligible approvers section. +- By GitLab Bot to ping reviewers for community contributions. + +GitLab cannot automatically associate the stage and group metadata in our documentation +pages with the technical writer assigned to that group, so we use a Rake task to +generate entries for the `CODEOWNERS` file. Declaring code owners for pages reduces +the number of times GitLab Bot pings the entire Technical Writing team. + +The `tw:codeowners` Rake task, located in [`lib/tasks/gitlab/tw/codeowners.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake), +contains an array of groups and their assigned technical writer. This task: + +- Outputs a line for each doc with metadata that matches a group in `lib/tasks/gitlab/tw/codeowners.rake`. + Files not matching a group are skipped. +- Adds the full path to the page, and the assigned technical writer. + +To prepare an update to the `CODEOWNERS` file: + +1. Update `lib/tasks/gitlab/tw/codeowners.rake` with the latest [TW team assignments](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments). + Make this update in a standalone merge request, as it runs a long pipeline and + requires backend maintainer review. Make sure this is merged before you update + `CODEOWNERS` in another merge request. +1. Run the task from the root directory of the `gitlab` repository, and save the output in a file: + + ```ruby + bundle exec rake tw:codeowners > ~/Desktop/updates.md + ``` + +1. Open the file you just created (`~/Desktop/updates.md` in this example), and prepare the output: + - Find and replace `./` with `/`. + - Sort the lines in alphabetical (ascending) order. If you use VS Code, you can + select everything, press <kbd>F1</kbd>, type `sort`, and select **Sort lines (ascending, case insensitive**. +1. Create a new branch for your `CODEOWNERS` updates. +1. Replace the documentation-related lines in the `^[Documentation Pages]` section + with the output you prepared. + + WARNING: + The documentation section is not the last section of the `CODEOWNERS` file. Don't + delete data that isn't ours! + +1. Create a commit with the raw changes. +1. From the command line, run `git diff master`. +1. In the diff, look for directory-level assignments to manually restore to the + `CODEOWNERS` file. If all files in a single directory are assigned to the same + technical writer, we simplify these entries. Remove all the lines for the individual + files, and leave a single entry for the directory, for example: `/doc/directory/ @tech.writer`. +1. In the diff, look for changes that don't match your expectations: + - New pages, or newly moved pages, show up as added lines. + - Deleted pages, and pages that are now redirects, show up as deleted lines. + - If you see an unusual number of changes to pages that all seem related, + check the metadata for the pages. A group might have been renamed and the Rake task + must be updated to match. +1. Create another commit with your manual changes, and create a second merge request + with your changes to the `CODEOWNERS` file. Assign it to a technical writing manager for review. + ## Move, rename, or delete a page See [redirects](redirects.md). diff --git a/doc/development/documentation/restful_api_styleguide.md b/doc/development/documentation/restful_api_styleguide.md index 0a24f9b67be..1f270a2b5ee 100644 --- a/doc/development/documentation/restful_api_styleguide.md +++ b/doc/development/documentation/restful_api_styleguide.md @@ -41,12 +41,18 @@ Use the following template to help you get started. Be sure to list any required attributes first in the table. ````markdown -## Descriptive title +## API name > Version history note. One or two sentence description of what endpoint does. +### Method title + +> Version history note. + +Description of the method. + ```plaintext METHOD /endpoint ``` @@ -83,8 +89,40 @@ Example response: ``` ```` -Adjust the [version history note accordingly](versions.md#add-a-version-history-item) -to describe the GitLab release that introduced the API call. +## Version history + +Add [version history](versions.md#documenting-version-specific-features) +to describe new or updated API calls. + +To add version history for an individual attribute, include it in the version history +for the section. For example: + +```markdown +### Edit a widget + +> `widget_message` [introduced](<link-to-issue>) in GitLab 14.3. +``` + +## Attribute deprecation + +To deprecate an attribute: + +1. Add a version history note. + + ```markdown + > - `widget_name` [deprecated](<link-to-issue>) in GitLab 14.7. + ``` + +1. Add inline deprecation text to the description. + + ```markdown + | Attribute | Type | Required | Description | + |:--------------|:-------|:-----------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| + | `widget_name` | string | **{dotted-circle}** No | [Deprecated](<link-to-issue>) in GitLab 14.7 and is planned for removal in 15.4. Use `widget_id` instead. The name of the widget. | + ``` + +1. Optional. To widely announce the change, or if it's a breaking change, + [update the deprecations and removals documentation](../deprecation_guidelines/#update-the-deprecations-and-removals-documentation). ## Method description diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 6d2b93b9462..e1e0da03abc 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -103,7 +103,7 @@ The global nav has five levels: - Doc - Doc -You can view this structure in [the navigation.yml file](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/content/_data/navigation.yaml). +You can view this structure in [the `navigation.yml` file](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/content/_data/navigation.yaml). **Do not** [add items](#add-a-navigation-entry) to the global nav without the consent of one of the technical writers. diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index 3566ab82379..05015fe7c5f 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -114,12 +114,12 @@ pipeline in the main `gitlab` repository as well as in `gitlab-docs`. Create an a different name first and test it to ensure you do not break the pipelines. 1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI/CD > Pipelines**. -1. Click the **Run pipeline** button. +1. Select **Run pipeline**. 1. See that a new pipeline is running. The jobs that build the images are in the first - stage, `build-images`. You can click the pipeline number to see the larger pipeline - graph, or click the first (`build-images`) stage in the mini pipeline graph to + stage, `build-images`. You can select the pipeline number to see the larger pipeline + graph, or select the first (`build-images`) stage in the mini pipeline graph to expose the jobs that build the images. -1. Click the **play** (**{play}**) button next to the images you want to rebuild. +1. Select the **play** (**{play}**) button next to the images you want to rebuild. - Normally, you do not need to rebuild the `image:gitlab-docs-base` image, as it rarely changes. If it does need to be rebuilt, be sure to only run `image:docs-lint` after it is finished rebuilding. @@ -133,7 +133,7 @@ and deploys it to <https://docs.gitlab.com>. To build and deploy the site immediately (must have the Maintainer role): 1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI/CD > Schedules**. -1. For the `Build docs.gitlab.com every 4 hours` scheduled pipeline, click the **play** (**{play}**) button. +1. For the `Build docs.gitlab.com every 4 hours` scheduled pipeline, select the **play** (**{play}**) button. Read more about [documentation deployments](deployment_process.md). diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index 329fd279b99..a02046d4466 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -37,9 +37,6 @@ Don't tell them **how** to do this thing. Tell them **what it is**. If you start describing another concept, start a new concept and link to it. -Also, do not use **Overview** or **Introduction** for the title. Instead, -use a noun or phrase that someone would search for. - Concepts should be in this format: ```markdown @@ -53,6 +50,19 @@ Remember, if you start to describe about another concept, stop yourself. Each concept should be about one concept only. ``` +### Concept headings + +For the heading text, use a noun. For example, `Widgets` or `GDK dependency management`. + +If a noun is ambiguous, you can add a gerund. For example, `Documenting versions` instead of `Versions`. + +Avoid these heading titles: + +- `Overview` or `Introduction`. Instead, use a more specific + noun or phrase that someone would search for. +- `Use cases`. Instead, incorporate the information as part of the concept. +- `How it works`. Instead, use a noun followed by `workflow`. For example, `Merge request workflow`. + ## Task A task gives instructions for how to complete a procedure. @@ -101,8 +111,13 @@ To create an issue: The issue is created. You can view it by going to **Issues > List**. ``` +### Task headings + +For the heading text, use the structure `active verb` + `noun`. +For example, `Create an issue`. + If you have several tasks on a page that share prerequisites, you can use the title -**Prerequisites**, and link to it. +`Prerequisites` and link to it. ## Reference @@ -119,8 +134,17 @@ Introductory sentence. | **Name** | Descriptive sentence about the setting. | ``` -If a feature or concept has its own prerequisites, you can use reference -content to create a **Prerequisites** header for the information. +### Reference headings + +Reference headings are usually nouns. + +Avoid these heading titles: + +- `Important notes`. Instead, incorporate this information + closer to where it belongs. For example, this information might be a prerequisite + for a task, or information about a concept. +- `Limitations`. Instead, move the content near other similar information. + If you must, you can use the title `Known issues`. ## Troubleshooting @@ -142,6 +166,10 @@ This issue occurs when... The workaround is... ``` +If multiple causes or workarounds exist, consider putting them into a table format. + +### Troubleshooting headings + For the heading: - Consider including at least a partial error message. @@ -149,7 +177,17 @@ For the heading: If you do not put the full error in the title, include it in the body text. -If multiple causes or workarounds exist, consider putting them into a table format. +## General heading text guidelines + +In general, for heading text: + +- Be clear and direct. Make every word count. +- Use articles and prepositions. +- Follow [capitalization](styleguide/index.md#capitalization) guidelines. +- Do not repeat text from earlier headings. For example, if the page is about merge requests, + instead of `Troubleshooting merge requests`, use only `Troubleshooting`. + +See also [guidelines for headings in Markdown](styleguide/index.md#headings-in-markdown). ## Other types of content diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index c11d1422167..700d64c30d1 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -86,7 +86,7 @@ move in this direction, so we can address these issues: information into a format that is geared toward helping others, rather than documenting how a feature was implemented. -GitLab uses these [topic type templates](../structure.md). +GitLab uses these [topic types](../structure.md). ### Link instead of repeating text @@ -143,6 +143,25 @@ Hard-coded HTML is valid, although it's discouraged from being used. HTML is per - Special styling is required. - Reviewed and approved by a technical writer. +### Headings in Markdown + +Each documentation page begins with a level 1 heading (`#`). This becomes the `h1` element when +the page is rendered to HTML. There can be only **one** level 1 heading per page. + +- For each subsection, increment the heading level. In other words, increment the number of `#` characters + in front of the heading. +- Do not skip a level. For example: `##` > `####`. +- Leave one blank line before and after the heading. + +When you change heading text, the anchor link changes. To avoid broken links: + +- Do not use step numbers in headings. +- When possible, do not use words that might change in the future. + +Also, do not use links as part of heading text. + +See also [heading guidelines for specific topic types](../structure.md). + ### Markdown Rules GitLab ensures that the Markdown used across all documentation is consistent, as @@ -191,6 +210,8 @@ GitLab documentation should be clear and easy to understand. ### Capitalization +As a company, we tend toward lowercase. + #### Headings Use sentence case. For example: @@ -220,7 +241,7 @@ create an issue or an MR to propose a change to the user interface text. If the term is not in the word list, ask a GitLab Technical Writer for advice. Do not match the capitalization of terms or phrases on the [Features page](https://about.gitlab.com/features/) -or [features.yml](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/features.yml) +or [`features.yml`](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/features.yml) by default. #### Other terms @@ -589,6 +610,10 @@ Consider installing a plugin or extension in your editor for formatting tables: - [Markdown Table Formatter](https://packagecontrol.io/packages/Markdown%20Table%20Formatter) for Sublime Text - [Markdown Table Formatter](https://atom.io/packages/markdown-table-formatter) for Atom +### Table headings + +Use sentence case for table headings. For example, `Keyword value` or `Project name`. + ### Feature tables When creating tables of lists of features (such the features @@ -642,45 +667,6 @@ For other punctuation rules, refer to the [Pajamas Design System Punctuation section](https://design.gitlab.com/content/punctuation/). This is overridden by the [documentation-specific punctuation rules](#punctuation). -## Headings - -In the Markdown document: - -- Add one H1 (`#`) at the start of the page. The `h1` becomes the document `<title>`. -- After the H1, follow the order `h2` > `h3` > `h4` > `h5` > `h6`. -- Do not skip a level. For example: `h2` > `h4`. -- Leave one blank line before and after the heading. - -For the heading text, **do**: - -- Be clear and direct. Make every word count. -- Use active, imperative verbs for [tasks](../structure.md#task). For example, `Create an issue`. -- Use `ing` (gerund) verbs only when you need a topic that introduces tasks. For example, `Configuring GDK`. -- Use nouns for [concepts](../structure.md#concept). For example, `GDK dependency management`. If a noun is - ambiguous, you can add a gerund. For example, `Documenting versions` instead of `Versions`. -- Talk about what the product does, realistically but from a positive perspective. Instead of - `Limitations`, move the content near other similar information. If you must, you can - use the title `Known issues`. -- Use articles and prepositions. -- Add the [product badge](#product-tier-badges) that corresponds to the license tier. -- Follow [capitalization](#capitalization) guidelines. - -For the heading text, **do not**: - -- Use generic words like `Overview` or `Use cases`. Instead, incorporate - the information under a concept heading. -- Use `How it works`. Incorporate this information under a concept, or use a - noun followed by `workflow`. For example, `Merge request workflow`. -- Use `Important Notes`. Incorporate this information closer to where it belongs. -- Use numbers to indicate steps. If the numbers change, the anchor links changes, - which eventually leads to dead links. If you think you must add numbers in headings, - at least discuss it with a writer in the merge request. -- Use words that might change in the future. Changing - a heading changes its anchor URL, which affects other linked pages. -- Repeat text from earlier headings. For example, instead of `Troubleshooting merge requests`, - use `Troubleshooting`. -- Use links. - ### Anchor links Headings generate anchor links when rendered. `## This is an example` generates @@ -1193,7 +1179,7 @@ This is how it renders on the GitLab documentation site: > Notes: > -> - The `figure` tag is required for semantic SEO and the `video_container` +> - The `figure` tag is required for semantic SEO and the `video-container` class is necessary to make sure the video is responsive and displays on different mobile devices. > - The `<div class="video-fallback">` is a fallback necessary for diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index e7d927de2cf..c753c39b727 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -139,6 +139,16 @@ Do not use **and so on**. Instead, be more specific. For details, see Use [**section**](#section) instead of **area**. The only exception is [the Admin Area](#admin-area). +## associate + +Do not use **associate** when describing adding issues to epics, or users to issues, merge requests, +or epics. + +Instead, use **assign**. For example: + +- Assign the issue to an epic. +- Assign a user to the issue. + ## below Try to avoid **below** when referring to an example or table in a documentation page. If required, use **following** instead. For example: @@ -347,6 +357,8 @@ See also [**type**](#type). Use lowercase for **epic**. +See also [associate](#associate). + ## epic board Use lowercase for **epic board**. @@ -395,6 +407,13 @@ of the fields at once. For example: Learn more about [documenting multiple fields at once](index.md#documenting-multiple-fields-at-once). +## filter + +When you are viewing a list of items, like issues or merge requests, you filter the list by +the available attributes. For example, you might filter by assignee or reviewer. + +Filtering is different from [searching](#search). + ## foo Do not use **foo** in product documentation. You can use it in our API and contributor documentation, but try to use a clearer and more meaningful example instead. @@ -851,6 +870,13 @@ Do not use **scalability** when talking about increasing GitLab performance for are sometimes acceptable, but references to increasing GitLab performance for additional users should direct readers to the GitLab [reference architectures](../../../administration/reference_architectures/index.md) page. +## search + +When you search, you type a string in the search box on the top bar. +The search results are displayed on a search page. + +Searching is different from [filtering](#filter). + ## section Use **section** to describe an area on a page. For example, if a page has lines that separate the UI diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index 81e1eca8724..feb10845aea 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -276,6 +276,22 @@ guidelines: | UI text from GitLab | Verify it correctly matches the UI, then: If it does not match the UI, update it. If it matches the UI, but the UI seems incorrect, create an issue to see if the UI needs to be fixed. If it matches the UI and seems correct, add it to the [vale spelling exceptions list](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/spelling-exceptions.txt). | | UI text from a third-party product | Rewrite the sentence to avoid it, or [add the vale exception code in-line](#disable-vale-tests). | +#### Vale uppercase (acronym) test + +The [`Uppercase.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Uppercase.yml) +test checks for incorrect usage of words in all capitals. For example, avoid usage +like `This is NOT important`. + +If the word must be in all capitals, follow these guidelines: + +| Flagged word | Guideline | +|----------------------------------------------------------------|-----------| +| Acronym (likely known by the average visitor to that page) | Add the acronym to the list of words and acronyms in `Uppercase.yml`. | +| Acronym (likely not known by the average visitor to that page) | The first time the acronym is used, write it out fully followed by the acronym in parentheses. In later uses, use just the acronym by itself. For example: `This feature uses the File Transfer Protocol (FTP). FTP is...`. | +| Correctly capitalized name of a product or service | Add the name to the list of words and acronyms in `Uppercase.yml`. | +| Command, variable, code, or similar | Put it in backticks or a code block. For example: ``Use `FALSE` as the variable value.`` | +| UI text from a third-party product | Rewrite the sentence to avoid it, or [add the vale exception code in-line](#disable-vale-tests). | + #### Vale readability score In [`ReadingLevel.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/ReadingLevel.yml), @@ -321,12 +337,12 @@ To match the versions of `markdownlint-cli` and `vale` used in the GitLab projec [versions used (see `variables:` section)](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/.gitlab-ci.yml) when building the `image:docs-lint-markdown` Docker image containing these tools for CI/CD. -| Tool | Version | Command | Additional information | -|--------------------|-----------|-------------------------------------------|------------------------| -| `markdownlint-cli` | Latest | `yarn global add markdownlint-cli` | n/a | -| `markdownlint-cli` | Specific | `yarn global add markdownlint-cli@0.23.2` | The `@` indicates a specific version, and this example updates the tool to version `0.23.2`. | -| Vale | Latest | `brew update && brew upgrade vale` | This command is for macOS only. | -| Vale | Specific | n/a | Not possible using `brew`, but can be [directly downloaded](https://github.com/errata-ai/vale/releases). | +| Tool | Version | Command | Additional information | +|--------------------|-----------|--------------------------------------------------------|------------------------| +| `markdownlint-cli` | Latest | `yarn global add markdownlint-cli` | None. | +| `markdownlint-cli` | Specific | `yarn global add markdownlint-cli@0.23.2` | The `@` indicates a specific version, and this example updates the tool to version `0.23.2`. | +| Vale | Latest | `brew update && brew upgrade vale` | This command is for macOS only. | +| Vale | Specific | Not applicable. | Binaries can be [directly downloaded](https://github.com/errata-ai/vale/releases). | ### Configure editors diff --git a/doc/development/documentation/versions.md b/doc/development/documentation/versions.md index 0f2bdca4c73..067c37d30aa 100644 --- a/doc/development/documentation/versions.md +++ b/doc/development/documentation/versions.md @@ -25,7 +25,7 @@ To view versions that are not available on `docs.gitlab.com`: ## Documenting version-specific features When a feature is added or updated, you can include its version information -either as a **Version history** bullet or as an inline text reference. +either as a **Version history** list item or as an inline text reference. You do not need to add version information on the pages in the `/development` directory. @@ -132,7 +132,7 @@ To remove a page: ```markdown --- - stage: Enablement + stage: Data Stores group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments remove_date: '2022-08-02' @@ -141,8 +141,8 @@ To remove a page: # Title (removed) **(ULTIMATE SELF)** - This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8 - and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in 15.0. + This feature was [deprecated](<link-to-issue>) in GitLab X.Y + and [removed](<link-to-issue>) in X.Y. Use [feature X](<link-to-issue>) instead. ``` @@ -162,12 +162,12 @@ To remove a topic: For the `remove_date`, set a date three months after the release where it was removed. ```markdown - <!--- start_remove The following content will be removed on remove_date: '2023-08-22' --> + <!--- start_remove The following content will be removed on remove_date: 'YYYY-MM-DD' --> ## Title (removed) **(ULTIMATE SELF)** - This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in GitLab 14.8 - and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/351963) in 15.0. + This feature was [deprecated](<link-to-issue>) in GitLab X.Y + and [removed](<link-to-issue>) in X.Y. Use [feature X](<link-to-issue>) instead. <!--- end_remove --> @@ -179,8 +179,8 @@ This content is removed from the documentation as part of the Technical Writing ## Which versions are removed GitLab supports the current major version and two previous major versions. -For example, if 14.0 is the current major version, all major and minor releases of -GitLab 14.0, 13.0 and 12.0 are supported. +For example, if 15.0 is the current major version, all major and minor releases of +GitLab 15.0, 14.0, and 13.0 are supported. [View the list of supported versions](https://about.gitlab.com/support/statement-of-support.html#version-support). diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 7c67b3495ba..d32ceb43ce9 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -9,17 +9,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w This area is to maintain a compendium of useful information when working with Elasticsearch. Information on how to enable Elasticsearch and perform the initial indexing is in -the [Elasticsearch integration documentation](../integration/elasticsearch.md#enable-advanced-search). +the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search). ## Deep Dive -In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. +In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3. ## Supported Versions -See [Version Requirements](../integration/elasticsearch.md#version-requirements). +See [Version Requirements](../integration/advanced_search/elasticsearch.md#version-requirements). Developers making significant changes to Elasticsearch queries should test their features against all our supported versions. @@ -69,7 +69,7 @@ The `whitespace` tokenizer was selected in order to have more control over how t Please see the `code` filter for an explanation on how tokens are split. NOTE: -The [Elasticsearch code_analyzer doesn't account for all code cases](../integration/elasticsearch.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). +The [Elasticsearch code_analyzer doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). #### `code_search_analyzer` diff --git a/doc/development/event_store.md b/doc/development/event_store.md index afd5640271e..fa7208ead04 100644 --- a/doc/development/event_store.md +++ b/doc/development/event_store.md @@ -316,11 +316,11 @@ RSpec.describe MergeRequests::UpdateHeadPipelineWorker do let(:pipeline_created_event) { Ci::PipelineCreatedEvent.new(data: ({ pipeline_id: pipeline.id })) } # This shared example ensures that an event is published and correctly processed by - # the current subscriber (`described_class`). + # the current subscriber (`described_class`). It also ensures that the worker is idempotent. it_behaves_like 'subscribes to event' do let(:event) { pipeline_created_event } end - + it 'does something' do # This helper directly executes `perform` ensuring that `handle_event` is called correctly. consume_event(subscriber: described_class, event: pipeline_created_event) diff --git a/doc/development/experiment_guide/experiment_code_reviews.md b/doc/development/experiment_guide/experiment_code_reviews.md index fdde89caa34..eda316db9d4 100644 --- a/doc/development/experiment_guide/experiment_code_reviews.md +++ b/doc/development/experiment_guide/experiment_code_reviews.md @@ -22,4 +22,4 @@ but is acceptable for now, mention your concerns with a note that there's no need to change the code. The author can then add a comment to this piece of code and link to the issue that resolves the experiment. The author or reviewer can add a link to this concern in the experiment rollout issue under the `Experiment Successful Cleanup Concerns` section of the description. -If the experiment is successful and becomes part of the product, any items that appear under this section will be addressed. +If the experiment is successful and becomes part of the product, any items that appear under this section are addressed. diff --git a/doc/development/experiment_guide/experiment_rollout.md b/doc/development/experiment_guide/experiment_rollout.md index afa32d75221..ff0844f9d3c 100644 --- a/doc/development/experiment_guide/experiment_rollout.md +++ b/doc/development/experiment_guide/experiment_rollout.md @@ -12,7 +12,7 @@ Each experiment should have an [experiment rollout](https://gitlab.com/groups/gi The rollout issue is similar to a feature flag rollout issue, and is also used to track the status of an experiment. When an experiment is deployed, the due date of the issue should be set (this depends on the experiment but can be up to a few weeks in the future). -After the deadline, the issue needs to be resolved and either: +After the deadline, the issue must be resolved and either: - It was successful and the experiment becomes the new default. - It was not successful and all code related to the experiment is removed. @@ -29,7 +29,7 @@ This can be done via ChatOps: - [disable](../feature_flags/controls.md#disabling-feature-flags): `/chatops run feature set gitlab_experiment false` - [enable](../feature_flags/controls.md#process): `/chatops run feature delete gitlab_experiment` -- This allows the `default_enabled` [value of true in the yml](https://gitlab.com/gitlab-org/gitlab/-/blob/016430f6751b0c34abb24f74608c80a1a8268f20/config/feature_flags/ops/gitlab_experiment.yml#L8) to be honored. +- This allows the `default_enabled` [value of true in the YAML](https://gitlab.com/gitlab-org/gitlab/-/blob/016430f6751b0c34abb24f74608c80a1a8268f20/config/feature_flags/ops/gitlab_experiment.yml#L8) to be honored. ## Notes on feature flags @@ -42,8 +42,8 @@ You may already be familiar with the concept of feature flags in GitLab, but usi feature flags in experiments is a bit different. While in general terms, a feature flag is viewed as being either `on` or `off`, this isn't accurate for experiments. -Generally, `off` means that when we ask if a feature flag is enabled, it will always -return `false`, and `on` means that it will always return `true`. An interim state, +Generally, `off` means that when we ask if a feature flag is enabled, it always +returns `false`, and `on` means that it always returns `true`. An interim state, considered `conditional`, also exists. We take advantage of this trinary state of feature flags. To understand this `conditional` aspect: consider that either of these settings puts a feature flag into this state: @@ -64,7 +64,7 @@ We don't refer to this as being enabled, because that's a confusing and overload term here. In the experiment terms, our experiment is _running_, and the feature flag is `conditional`. -When a feature flag is enabled (meaning the state is `on`), the candidate will always be +When a feature flag is enabled (meaning the state is `on`), the candidate is always assigned. We should try to be consistent with our terms, and so for experiments, we have an diff --git a/doc/development/experiment_guide/implementing_experiments.md b/doc/development/experiment_guide/implementing_experiments.md index 3c33d015108..c9e277873dc 100644 --- a/doc/development/experiment_guide/implementing_experiments.md +++ b/doc/development/experiment_guide/implementing_experiments.md @@ -13,14 +13,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w Start by generating a feature flag using the `bin/feature-flag` command as you normally would for a development feature flag, making sure to use `experiment` for the type. For the sake of documentation let's name our feature flag (and experiment) -"pill_color". +`pill_color`. ```shell bin/feature-flag pill_color -t experiment ``` After you generate the desired feature flag, you can immediately implement an -experiment in code. An experiment implementation can be as simple as: +experiment in code. A basic experiment implementation can be: ```ruby experiment(:pill_color, actor: current_user) do |e| @@ -30,8 +30,8 @@ experiment(:pill_color, actor: current_user) do |e| end ``` -When this code executes, the experiment is run, a variant is assigned, and (if within a -controller or view) a `window.gl.experiments.pill_color` object will be available in the +When this code executes, the experiment is run, a variant is assigned, and (if in a +controller or view) a `window.gl.experiments.pill_color` object is available in the client layer, with details like: - The assigned variant. @@ -102,7 +102,7 @@ contexts to simplify reporting: - `{ actor: current_user }`: Assigns a variant and is "sticky" to each user (or "client" if `current_user` is nil) who enters the experiment. -- `{ project: project }`: Assigns a variant and is "sticky" to the project currently +- `{ project: project }`: Assigns a variant and is "sticky" to the project being viewed. If running your experiment is more useful when viewing a project, rather than when a specific user is viewing any project, consider this approach. - `{ group: group }`: Similar to the project example, but applies to a wider @@ -151,7 +151,7 @@ wouldn't be resolvable. There are two ways to implement an experiment: -1. The simple experiment style described previously. +1. The basic experiment style described previously. 1. A more advanced style where an experiment class is provided. The advanced style is handled by naming convention, and works similar to what you @@ -224,8 +224,8 @@ end When an experiment runs, the segmentation rules are executed in the order they're defined. The first segmentation rule to produce a truthy result assigns the variant. -In our example, any user named `'Richard'`, regardless of account age, will always -be assigned the _red_ variant. If you want the opposite logic, flip the order. +In our example, any user named `'Richard'`, regardless of account age, is always +assigned the _red_ variant. If you want the opposite logic, flip the order. NOTE: Keep in mind when defining segmentation rules: after a truthy result, the remaining @@ -275,7 +275,7 @@ end One of the most important aspects of experiments is gathering data and reporting on it. You can use the `track` method to track events across an experimental implementation. You can track events consistently to an experiment if you provide the same context between -calls to your experiment. If you do not yet understand context, you should read +calls to your experiment. If you do not understand context, you should read about contexts now. We can assume we run the experiment in one or a few places, but @@ -295,7 +295,7 @@ experiment have a special added to the event. This can be used - typically by the data team - to create a connection between the events on a given experiment. -If our current user hasn't encountered the experiment yet (meaning where the experiment +If our user hasn't encountered the experiment (meaning where the experiment is run), and we track an event for them, they are assigned a variant and see that variant if they ever encountered the experiment later, when an `:assignment` event would be tracked at that time for them. @@ -316,9 +316,9 @@ so it can be used when resolving experimentation in the client layer. Given that we've defined a class for our experiment, and have defined the variants for it, we can publish that experiment in a couple ways. -The first way is simply by running the experiment. Assuming the experiment has been run, it will surface in the client layer without having to do anything special. +The first way is by running the experiment. Assuming the experiment has been run, it surfaces in the client layer without having to do anything special. -The second way doesn't run the experiment and is intended to be used if the experiment only needs to surface in the client layer. To accomplish this we can simply `.publish` the experiment. This won't run any logic, but does surface the experiment details in the client layer so they can be utilized there. +The second way doesn't run the experiment and is intended to be used if the experiment must only surface in the client layer. To accomplish this we can `.publish` the experiment. This does not run any logic, but does surface the experiment details in the client layer so they can be utilized there. An example might be to publish an experiment in a `before_action` in a controller. Assuming we've defined the `PillColorExperiment` class, like we have above, we can surface it to the client by publishing it instead of running it: @@ -329,7 +329,7 @@ before_action -> { experiment(:pill_color).publish }, only: [:show] You can then see this surface in the JavaScript console: ```javascript -window.gl.experiments // => { pill_color: { excluded: false, experiment: "pill_color", key: "ca63ac02", variant: "candidate" } } +window.gl.experiments // => { pill_color: { excluded: false, experiment: "pill_color", key: "ca63ac02", variant: "candidate" } } ``` ### Using experiments in Vue @@ -366,4 +366,4 @@ export default { ``` NOTE: -When there is no experiment data in the `window.gl.experiments` object for the given experiment name, the `control` slot will be used, if it exists. +When there is no experiment data in the `window.gl.experiments` object for the given experiment name, the `control` slot is used, if it exists. diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index b140cce34fc..163cd009c51 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Experiment Guide -Experiments can be conducted by any GitLab team, most often the teams from the -[Growth Sub-department](https://about.gitlab.com/handbook/engineering/development/growth/). +Experiments can be conducted by any GitLab team, most often the teams from the +[Growth Sub-department](https://about.gitlab.com/handbook/engineering/development/growth/). Experiments are not tied to releases because they primarily target GitLab.com. Experiments are run as an A/B/n test, and are behind an [experiment feature flag](../feature_flags/#experiment-type) @@ -27,7 +27,7 @@ sometimes referred to as GLEX, to run our experiments. The gem exists in a separ so it can be shared across any GitLab property that uses Ruby. You should feel comfortable reading the documentation on that project if you want to dig into more advanced topics or open issues. Be aware that the documentation there reflects what's in the main branch and may not be the same as -the version being used within GitLab. +the version being used in GitLab. ## Glossary of terms @@ -43,7 +43,7 @@ when communicating about experiments: ## Implementing an experiment -[`GLEX`](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment) - or `Gitlab::Experiment`, the `gitlab-experiment` gem - is the preferred option for implementing an experiment in GitLab. +[GLEX](https://gitlab.com/gitlab-org/ruby/gems/gitlab-experiment) - or `Gitlab::Experiment`, the `gitlab-experiment` gem - is the preferred option for implementing an experiment in GitLab. For more information, see [Implementing an A/B/n experiment using GLEX](implementing_experiments.md). diff --git a/doc/development/experiment_guide/testing_experiments.md b/doc/development/experiment_guide/testing_experiments.md index 08ff91a3deb..a73896c8436 100644 --- a/doc/development/experiment_guide/testing_experiments.md +++ b/doc/development/experiment_guide/testing_experiments.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Testing experiments with RSpec -In the course of working with experiments, you'll probably want to utilize the RSpec +In the course of working with experiments, you might want to use the RSpec tooling that's built in. This happens automatically for files in `spec/experiments`, but for other files and specs you want to include it in, you can specify the `:experiment` type: @@ -48,7 +48,7 @@ segmentations using the matchers. class ExampleExperiment < ApplicationExperiment control { } candidate { '_candidate_' } - + exclude { context.actor.first_name == 'Richard' } segment(variant: :candidate) { context.actor.username == 'jejacks0n' } end @@ -84,7 +84,7 @@ expect(subject).to track(:my_event) subject.track(:my_event) ``` -You can use the `on_next_instance` chain method to specify that it will happen +You can use the `on_next_instance` chain method to specify that it happens on the next instance of the experiment. This helps you if you're calling `experiment(:example).track` downstream: @@ -127,7 +127,7 @@ describe('when my_experiment is enabled', () => { ``` NOTE: -This method of stubbing in Jest specs will not automatically un-stub itself at the end of the test. We merge our stubbed experiment in with all the other global data in `window.gl`. If you need to remove the stubbed experiments after your test or ensure a clean global object before your test, you'll need to manage the global object directly yourself: +This method of stubbing in Jest specs does not automatically un-stub itself at the end of the test. We merge our stubbed experiment in with all the other global data in `window.gl`. If you must remove the stubbed experiments after your test or ensure a clean global object before your test, you must manage the global object directly yourself: ```javascript describe('tests that care about global state', () => { diff --git a/doc/development/export_csv.md b/doc/development/export_csv.md index 998e5b1fb3b..0f50d1438fc 100644 --- a/doc/development/export_csv.md +++ b/doc/development/export_csv.md @@ -15,7 +15,7 @@ This document lists the different implementations of CSV export in GitLab codeba | As email attachment | - Asynchronously process the query with background job.<br>- Email uses the export as an attachment. | - Asynchronous processing. | - Requires users use a different app (email) to download the CSV.<br>- Email providers may limit attachment size. | - [Export issues](../user/project/issues/csv_export.md)<br>- [Export merge requests](../user/project/merge_requests/csv_export.md) | | As downloadable link in email (*) | - Asynchronously process the query with background job.<br>- Email uses an export link. | - Asynchronous processing.<br>- Bypasses email provider attachment size limit. | - Requires users use a different app (email).<br>- Requires additional storage and cleanup. | [Export User Permissions](https://gitlab.com/gitlab-org/gitlab/-/issues/1772) | | Polling (non-persistent state) | - Asynchronously processes the query with the background job.<br>- Frontend(FE) polls every few seconds to check if CSV file is ready. | - Asynchronous processing.<br>- Automatically downloads to local machine on completion.<br>- In-app solution. | - Non-persistable request - request expires when user navigates to a different page.<br>- API is processed for each polling request. | [Export Vulnerabilities](../user/application_security/vulnerability_report/#export-vulnerability-details) | -| Polling (persistent state) (*) | - Asynchronously processes the query with background job.<br>- Backend (BE) maintains the export state<br>- FE polls every few seconds to check status.<br>- FE shows 'Download link' when export is ready.<br>- User can download or regenerate a new report. | - Asynchronous processing.<br>- No database calls made during the polling requests (HTTP 304 status is returned until export status changes).<br>- Does not require user to stay on page until export is complete.<br>- In-app solution.<br>- Can be expanded into a generic CSV feature (such as dashboard / CSV API). | - Requires to maintain export states in DB.<br>- Does not automatically download the CSV export to local machine, requires users to click 'Download' button. | [Export Merge Commits Report](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43055) | +| Polling (persistent state) (*) | - Asynchronously processes the query with background job.<br>- Backend (BE) maintains the export state<br>- FE polls every few seconds to check status.<br>- FE shows 'Download link' when export is ready.<br>- User can download or regenerate a new report. | - Asynchronous processing.<br>- No database calls made during the polling requests (HTTP 304 status is returned until export status changes).<br>- Does not require user to stay on page until export is complete.<br>- In-app solution.<br>- Can be expanded into a generic CSV feature (such as dashboard / CSV API). | - Requires to maintain export states in DB.<br>- Does not automatically download the CSV export to local machine, requires users to select 'Download'. | [Export Merge Commits Report](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43055) | NOTE: Export types marked as * are currently work in progress. diff --git a/doc/development/fe_guide/emojis.md b/doc/development/fe_guide/emojis.md index 7ef88c5ca19..3c7fc20440b 100644 --- a/doc/development/fe_guide/emojis.md +++ b/doc/development/fe_guide/emojis.md @@ -24,6 +24,10 @@ when your platform does not support it. 1. Ensure new sprite sheets generated for 1x and 2x - `app/assets/images/emoji.png` - `app/assets/images/emoji@2x.png` + 1. Update `fixtures/emojis/intents.json` with any new emoji that we would like to highlight as having positive or negative intent. + - Positive intent should be set to `0.5`. + - Neutral intent can be set to `1`. This is applied to all emoji automatically so there is no need to set this explicitly. + - Negative intent should be set to `1.5`. 1. Ensure you see new individual images copied into `app/assets/images/emoji/` 1. Ensure you can see the new emojis and their aliases in the GitLab Flavored Markdown (GLFM) Autocomplete 1. Ensure you can see the new emojis and their aliases in the award emoji menu diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 1e8f7f5fb81..39c39894dac 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -187,12 +187,12 @@ Be sure to add these polyfills to `app/assets/javascripts/commons/polyfills.js`. To see what polyfills are being used: 1. Navigate to your merge request. -1. In the secondary menu below the title of the merge request, click **Pipelines**, then - click the pipeline you want to view, to display the jobs in that pipeline. -1. Click the [`compile-production-assets`](https://gitlab.com/gitlab-org/gitlab/-/jobs/641770154) job. -1. In the right-hand sidebar, scroll to **Job Artifacts**, and click **Browse**. -1. Click the **webpack-report** folder to open it, and click **index.html**. -1. In the upper left corner of the page, click the right arrow **{angle-right}** +1. In the secondary menu below the title of the merge request, select **Pipelines**, then + select the pipeline you want to view, to display the jobs in that pipeline. +1. Select the [`compile-production-assets`](https://gitlab.com/gitlab-org/gitlab/-/jobs/641770154) job. +1. In the right-hand sidebar, scroll to **Job Artifacts**, and select **Browse**. +1. Select the **webpack-report** folder to open it, and select **index.html**. +1. In the upper left corner of the page, select the right arrow **{chevron-lg-right}** to display the explorer. 1. In the **Search modules** field, enter `gitlab/node_modules/core-js` to see which polyfills are being loaded and where: diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 5cfdaff0448..67b53fa0299 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -416,8 +416,8 @@ query getLocalData { } ``` -Similar to resolvers, your `typePolicies` will execute when the `@client` query is used. However, -using `makeVar` will trigger every relevant active Apollo query to reactively update when the state +Similar to resolvers, your `typePolicies` execute when the `@client` query is used. However, +using `makeVar` triggers every relevant active Apollo query to reactively update when the state mutates. ```javascript @@ -462,7 +462,7 @@ export const createLocalState = () => { }; ``` -Pass the cache config to your Apollo Client: +Pass the cache configuration to your Apollo Client: ```javascript // index.js @@ -490,7 +490,7 @@ return new Vue({ }); ``` -Wherever used, the local query will update as the state updates thanks to the **reactive variable**. +Wherever used, the local query updates as the state updates thanks to the **reactive variable**. ### Using with Vuex @@ -522,7 +522,7 @@ of the backend. #### Implementing frontend queries and mutations ahead of the backend -In such case, the frontend will define GraphQL schemas or fields that do not correspond to any +In such case, the frontend defines GraphQL schemas or fields that do not correspond to any backend resolver yet. This is fine as long as the implementation is properly feature-flagged so it does not translate to public-facing errors in the product. However, we do validate client-side queries/mutations against the backend GraphQL schema with the `graphql-verify` CI job. @@ -535,7 +535,7 @@ The preferred approach is to use the `@client` directive on any new query, mutat isn't yet supported by the backend. Any entity with the directive is skipped by the `graphql-verify` validation job. -Additionally Apollo will attempt to resolve them client-side, which can be used in conjunction with +Additionally Apollo attempts to resolve them client-side, which can be used in conjunction with [Mocking API response with local Apollo cache](#mocking-api-response-with-local-apollo-cache). This provides a convenient way of testing your feature with fake data defined client-side. When opening a merge request for your changes, it can be a good idea to provide local resolvers as a @@ -550,7 +550,7 @@ GraphQL queries/mutations validation can be completely turned off for specific f paths to the [`config/known_invalid_graphql_queries.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/known_invalid_graphql_queries.yml) file, much like you would disable ESLint for some files via an `.eslintignore` file. -Bear in mind that any file listed in here will not be validated at all. So if you're only adding +Bear in mind that any file listed in here is not validated at all. So if you're only adding fields to an existing query, use the `@client` directive approach so that the rest of the query is still validated. diff --git a/doc/development/fe_guide/haml.md b/doc/development/fe_guide/haml.md index 803bb89118c..00096ce7fdc 100644 --- a/doc/development/fe_guide/haml.md +++ b/doc/development/fe_guide/haml.md @@ -39,6 +39,15 @@ For example: %span = s_('GroupSettings|Prevent members from sending invitations to groups outside of %{group} and its subgroups.').html_safe % { group: link_to_group(@group) } %p.help-text= prevent_sharing_groups_outside_hierarchy_help_text(@group) + + .form-group.gl-mb-3 + .gl-form-checkbox.custom-control.custom-checkbox + = f.check_box :lfs_enabled, checked: @group.lfs_enabled?, class: 'custom-control-input' + = f.label :lfs_enabled, class: 'custom-control-label' do + %span + = _('Allow projects within this group to use Git LFS') + = link_to sprite_icon('question-o'), help_page_path('topics/git/lfs/index') + %p.help-text= _('This setting can be overridden in each project.') ``` - After: @@ -50,6 +59,14 @@ For example: s_('GroupSettings|Prevent members from sending invitations to groups outside of %{group} and its subgroups.').html_safe % { group: link_to_group(@group) }, help_text: prevent_sharing_groups_outside_hierarchy_help_text(@group), checkbox_options: { disabled: !can_change_prevent_sharing_groups_outside_hierarchy?(@group) } + + .form-group.gl-mb-3 + = f.gitlab_ui_checkbox_component :lfs_enabled, checkbox_options: { checked: @group.lfs_enabled? } do |c| + = c.label do + = _('Allow projects within this group to use Git LFS') + = link_to sprite_icon('question-o'), help_page_path('topics/git/lfs/index') + = c.help_text do + = _('This setting can be overridden in each project.') ``` ### Available components @@ -67,16 +84,27 @@ Currently only the listed components are available but more components are plann [GitLab UI Docs](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/base-form-form-checkbox--default) +##### Arguments + | Argument | Description | Type | Required (default value) | |---|---|---|---| | `method` | Attribute on the object passed to `gitlab_ui_form_for`. | `Symbol` | `true` | -| `label` | Checkbox label. | `String` | `true` | -| `help_text` | Help text displayed below the checkbox. | `String` | `false` (`nil`) | +| `label` | Checkbox label. `label` slot can be used instead of this argument if HTML is needed. | `String` | `false` (`nil`) | +| `help_text` | Help text displayed below the checkbox. `help_text` slot can be used instead of this argument if HTML is needed. | `String` | `false` (`nil`) | | `checkbox_options` | Options that are passed to [Rails `check_box` method](https://api.rubyonrails.org/classes/ActionView/Helpers/FormBuilder.html#method-i-check_box). | `Hash` | `false` (`{}`) | | `checked_value` | Value when checkbox is checked. | `String` | `false` (`'1'`) | | `unchecked_value` | Value when checkbox is unchecked. | `String` | `false` (`'0'`) | | `label_options` | Options that are passed to [Rails `label` method](https://api.rubyonrails.org/classes/ActionView/Helpers/FormBuilder.html#method-i-label). | `Hash` | `false` (`{}`) | +##### Slots + +This component supports [ViewComponent slots](https://viewcomponent.org/guide/slots.html). + +| Slot | Description +|---|---| +| `label` | Checkbox label content. This slot can be used instead of the `label` argument. | +| `help_text` | Help text content displayed below the checkbox. This slot can be used instead of the `help_text` argument. | + <!-- vale gitlab.Spelling = NO --> #### gitlab_ui_radio_component @@ -85,11 +113,22 @@ Currently only the listed components are available but more components are plann [GitLab UI Docs](https://gitlab-org.gitlab.io/gitlab-ui/?path=/story/base-form-form-radio--default) +##### Arguments + | Argument | Description | Type | Required (default value) | |---|---|---|---| | `method` | Attribute on the object passed to `gitlab_ui_form_for`. | `Symbol` | `true` | | `value` | The value of the radio tag. | `Symbol` | `true` | -| `label` | Radio label. | `String` | `true` | -| `help_text` | Help text displayed below the radio button. | `String` | `false` (`nil`) | +| `label` | Radio label. `label` slot can be used instead of this argument if HTML content is needed inside the label. | `String` | `false` (`nil`) | +| `help_text` | Help text displayed below the radio button. `help_text` slot can be used instead of this argument if HTML content is needed inside the help text. | `String` | `false` (`nil`) | | `radio_options` | Options that are passed to [Rails `radio_button` method](https://api.rubyonrails.org/classes/ActionView/Helpers/FormBuilder.html#method-i-radio_button). | `Hash` | `false` (`{}`) | | `label_options` | Options that are passed to [Rails `label` method](https://api.rubyonrails.org/classes/ActionView/Helpers/FormBuilder.html#method-i-label). | `Hash` | `false` (`{}`) | + +##### Slots + +This component supports [ViewComponent slots](https://viewcomponent.org/guide/slots.html). + +| Slot | Description +|---|---| +| `label` | Checkbox label content. This slot can be used instead of the `label` argument. | +| `help_text` | Help text content displayed below the radio button. This slot can be used instead of the `help_text` argument. | diff --git a/doc/development/fe_guide/registry_architecture.md b/doc/development/fe_guide/registry_architecture.md index 56d67e094b7..be14d5d920c 100644 --- a/doc/development/fe_guide/registry_architecture.md +++ b/doc/development/fe_guide/registry_architecture.md @@ -64,7 +64,7 @@ main pieces of the desired UI and UX of a registry page. The most important comp - `code-instruction`: represents a copyable box containing code. Supports multiline and single line code boxes. Snowplow tracks the code copy event. -- `details-row`: represents a row of details. Used to add additional info in the details area of +- `details-row`: represents a row of details. Used to add additional information in the details area of the `list-item` component. - `history-item`: represents a history list item used to build a timeline. - `list-item`: represents a list element in the registry. It supports: left action, left primary and diff --git a/doc/development/fe_guide/security.md b/doc/development/fe_guide/security.md index 79452327673..6f500c8f0fa 100644 --- a/doc/development/fe_guide/security.md +++ b/doc/development/fe_guide/security.md @@ -41,7 +41,7 @@ Security Policy headers in the GitLab Rails app. Some resources on implementing Content Security Policy: - [MDN Article on CSP](https://developer.mozilla.org/en-US/docs/Web/Security/CSP) -- [GitHub's CSP Journey on the GitHub Engineering Blog](http://githubengineering.com/githubs-csp-journey/) +- [GitHub's CSP Journey on the GitHub Engineering Blog](https://github.blog/2016-04-12-githubs-csp-journey/) - The Dropbox Engineering Blog's series on CSP: [1](https://blogs.dropbox.com/tech/2015/09/on-csp-reporting-and-filtering/), [2](https://blogs.dropbox.com/tech/2015/09/unsafe-inline-and-nonce-deployment/), [3](https://blogs.dropbox.com/tech/2015/09/csp-the-unexpected-eval/), [4](https://blogs.dropbox.com/tech/2015/09/csp-third-party-integrations-and-privilege-separation/) ### Subresource Integrity (SRI) @@ -59,7 +59,7 @@ All CSS and JavaScript assets should use Subresource Integrity. Some resources on implementing Subresource Integrity: - [MDN Article on SRI](https://developer.mozilla.org/en-us/docs/web/security/subresource_integrity) -- [Subresource Integrity on the GitHub Engineering Blog](http://githubengineering.com/subresource-integrity/) +- [Subresource Integrity on the GitHub Engineering Blog](https://github.blog/2015-09-19-subresource-integrity/) --> diff --git a/doc/development/fe_guide/storybook.md b/doc/development/fe_guide/storybook.md index 9c4bcf02389..4c0e7b2612b 100644 --- a/doc/development/fe_guide/storybook.md +++ b/doc/development/fe_guide/storybook.md @@ -53,6 +53,6 @@ To add a story: ## Mock backend APIs -GitLab’s Storybook uses [MirajeJS](https://miragejs.com/) to mock REST and GraphQL APIs. Storybook shares the MirajeJS server +The GitLab Storybook uses [MirajeJS](https://miragejs.com/) to mock REST and GraphQL APIs. Storybook shares the MirajeJS server with the [frontend integration tests](../testing_guide/testing_levels.md#frontend-integration-tests). You can find the MirajeJS configuration files in `spec/frontend_integration/mock_server`. diff --git a/doc/development/fe_guide/style/javascript.md b/doc/development/fe_guide/style/javascript.md index d04d1879476..d93dc8292d4 100644 --- a/doc/development/fe_guide/style/javascript.md +++ b/doc/development/fe_guide/style/javascript.md @@ -100,27 +100,31 @@ class a { ## Converting Strings to Integers -When converting strings to integers, `parseInt` has a slight performance advantage over `Number`, but `Number` is semantic and can be more readable. Prefer `parseInt`, but do not discourage `Number` if it significantly helps readability. +When converting strings to integers, `Number` is semantic and can be more readable. Both are allowable, but `Number` has a slight maintainability advantage. **WARNING:** `parseInt` **must** include the [radix argument](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/parseInt). ```javascript -// bad +// bad (missing radix argument) parseInt('10'); -// bad -things.map(parseInt) - -// ok -Number("106") +// good +parseInt("106", 10); // good -things.map(Number) +Number("106"); +``` + +```javascript +// bad (missing radix argument) +things.map(parseInt); // good -parseInt("106", 10) +things.map(Number); ``` +**PLEASE NOTE:** If the String could represent a non-integer (i.e., it includes a decimal), **do not** use `parseInt`. Consider `Number` or `parseFloat` instead. + ## CSS Selectors - Use `js-` prefix If a CSS class is only being used in JavaScript as a reference to the element, prefix diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index fecb0af936d..ae13e3fc8c5 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -583,7 +583,7 @@ This is the template for the example component which is tested in the data-testid="text-input" > <gl-button - variant="success" + variant="confirm" data-testid="add-button" @click="addTodo" >Add</gl-button> diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 68c14c1b0c9..07c3c83912a 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -95,6 +95,7 @@ Guidelines: - Consider notifying `#support_gitlab-com` beforehand. So in case if the feature has any side effects on user experience, they can mitigate and disable the feature flag to reduce some impact. - If the feature meets the requirements for creating a [Change Management](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#feature-flags-and-the-change-management-process) issue, create a Change Management issue per [criticality guidelines](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#change-request-workflows). - For simple, low-risk, easily reverted features, proceed and [enable the feature in `#production`](#process). +- For support requests to toggle feature flags for specific groups or projects, please follow the process outlined in the [support workflows](https://about.gitlab.com/handbook/support/workflows/saas_feature_flags.html). #### Process @@ -198,6 +199,14 @@ For groups the `--group` flag is available: /chatops run feature set --group=gitlab-org some_feature true ``` +Note that `--group` does not work with user namespaces. To enable a feature flag for a +generic namespace (including groups) use `--namespace`: + +```shell +/chatops run feature set --namespace=gitlab-org some_feature true +/chatops run feature set --namespace=myusername some_feature true +``` + Note that actor-based gates are applied before percentages. For example, considering the `group/project` as `gitlab-org/gitlab` and a given example feature as `some_feature`, if you run these 2 commands: @@ -215,6 +224,16 @@ actors. Feature.enabled?(:some_feature, group) ``` +Multiple actors can be passed together in a comma-separated form: + +```shell +/chatops run feature set --project=gitlab-org/gitlab,example-org/example-project some_feature true + +/chatops run feature set --group=gitlab-org,example-org some_feature true + +/chatops run feature set --namespace=gitlab-org,example-org some_feature true +``` + Lastly, to verify that the feature is deemed stable in as many cases as possible, you should fully roll out the feature by enabling the flag **globally** by running: @@ -267,7 +286,7 @@ To disable a feature flag that has been enabled for a specific project you can r You cannot selectively disable feature flags for a specific project/group/user without applying a [specific method of implementing](index.md#selectively-disable-by-actor) the feature flags. -If a feature flag is disabled via ChatOps, that will take precedence over the `default_enabled` value in the YML. In other words, you could have a feature enabled for on-premise installations but not for GitLab.com. +If a feature flag is disabled via ChatOps, that will take precedence over the `default_enabled` value in the YAML. In other words, you could have a feature enabled for on-premise installations but not for GitLab.com. ### Feature flag change logging diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 54158de6893..d21a46142a2 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -141,7 +141,7 @@ push_frontend_feature_flag(:my_ops_flag, project, type: :ops) An `experiment` feature flag should conform to the same standards as a `development` feature flag, although the interface has some differences. An experiment feature flag should have a rollout issue, -created using the [Experiment Tracking template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/experiment_tracking_template.md). More information can be found in the [experiment guide](../experiment_guide/index.md). +created using the [Experiment Tracking template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Experiment%20Rollout.md). More information can be found in the [experiment guide](../experiment_guide/index.md). ## Feature flag definition and validation @@ -226,6 +226,16 @@ Feature flags **must** be used in the MR that introduces them. Not doing so caus [broken master](https://about.gitlab.com/handbook/engineering/workflow/#broken-master) scenario due to the `rspec:feature-flags` job that only runs on the `master` branch. +## List all the feature flags + +To [use ChatOps](../../ci/chatops/index.md) to output all the feature flags in an environment to Slack, you can use the `run feature list` +command. For example: + +```shell +/chatops run feature list --dev +/chatops run feature list --staging +``` + ## Delete a feature flag See [cleaning up feature flags](controls.md#cleaning-up) for more information about diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index d4274c6275b..5b6f6ba0d98 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -1,6 +1,6 @@ --- -stage: none -group: unassigned +stage: Create +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -100,7 +100,7 @@ fips-mode-setup --disable #### Detect FIPS enablement in code -You can query `GitLab::FIPS` in Ruby code to determine if the instance is FIPS-enabled: +You can query `Gitlab::FIPS` in Ruby code to determine if the instance is FIPS-enabled: ```ruby def default_min_key_size(name) @@ -191,11 +191,11 @@ to ignore AMI changes. #### Ansible: Specify the FIPS Omnibus builds -The standard Omnibus GitLab releases build their own OpenSSL library, -which is not FIPS-validated. However, we have nightly builds that create -Omnibus packages that link against the operating system's OpenSSL library. To -use this package, update the `gitlab_repo_script_url` field in the -Ansible `vars.yml`. For example, you might modify +The standard Omnibus GitLab releases build their own OpenSSL library, which is +not FIPS-validated. However, we have nightly builds that create Omnibus packages +that link against the operating system's OpenSSL library. To use this package, +update the `gitlab_edition` and `gitlab_repo_script_url` fields in the Ansible +`vars.yml`. For example, you might modify `gitlab-environment-toolkit/ansible/environments/gitlab-10k/inventory/vars.yml` in this way: @@ -204,6 +204,7 @@ all: vars: ... gitlab_repo_script_url: "https://packages.gitlab.com/install/repositories/gitlab/nightly-fips-builds/script.deb.sh" + gitlab_edition: "gitlab-fips" ``` ### Cloud Native Hybrid @@ -300,7 +301,7 @@ all: gitlab_charts_custom_config_file: '/path/to/gitlab-environment-toolkit/ansible/environments/gitlab-10k/inventory/charts.yml' ``` -Now create `charts.yml` in the location specified above and specify tags with a `-ubi8` suffix. For example: +Now create `charts.yml` in the location specified above and specify tags with a `-fips` suffix. For example: ```yaml global: @@ -308,35 +309,38 @@ global: pullPolicy: Always certificates: image: - tag: master-ubi8 + tag: master-fips + kubectl: + image: + tag: master-fips gitlab: gitaly: image: - tag: master-ubi8 + tag: master-fips gitlab-exporter: image: - tag: master-ubi8 + tag: master-fips gitlab-shell: image: - tag: main-ubi8 # The default branch is main, not master + tag: main-fips # The default branch is main, not master gitlab-mailroom: image: - tag: master-ubi8 + tag: master-fips migrations: image: - tag: master-ubi8 + tag: master-fips sidekiq: image: - tag: master-ubi8 + tag: master-fips toolbox: image: - tag: master-ubi8 + tag: master-fips webservice: image: - tag: master-ubi8 + tag: master-fips workhorse: - tag: master-ubi8 + tag: master-fips nginx-ingress: controller: @@ -352,41 +356,44 @@ See [this issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3153#note_ how to build NGINX and the Ingress Controller. You can also use release tags, but the versioning is tricky because each -component may use its own versioning scheme. For example, for GitLab v14.10: +component may use its own versioning scheme. For example, for GitLab v15.1: ```yaml global: certificates: image: - tag: 20191127-r2-ubi8 + tag: 20211220-r0-fips + kubectl: + image: + tag: 1.18.20-fips gitlab: gitaly: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips gitlab-exporter: image: - tag: 11.14.0-ubi8 + tag: 11.15.2-fips gitlab-shell: image: - tag: v13.25.1-ubi8 + tag: v15.1.0-fips gitlab-mailroom: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips migrations: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips sidekiq: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips toolbox: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips webservice: image: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips workhorse: - tag: v14.10.0-ubi8 + tag: v15.1.0-fips ``` ## Verify FIPS @@ -508,3 +515,13 @@ the `webservice` container has the following tags: - `master` - `master-ubi8` - `master-fips` + +### Testing merge requests with a FIPS pipeline + +Merge requests that can trigger Package and QA, can trigger a FIPS package and a +Reference Architecture test pipeline. The base image used for the trigger is +Ubuntu 20.04 FIPS: + +1. Trigger `package-and-qa`, if not already triggered. +1. On the `gitlab-omnibus-mirror` child pipeline, manually trigger `Trigger:package:fips`. +1. When the package job is complete, manually trigger the `RAT:FIPS` job. diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md index c20c70623ae..77df6fbfb0d 100644 --- a/doc/development/foreign_keys.md +++ b/doc/development/foreign_keys.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -15,7 +15,7 @@ class User < ActiveRecord::Base end ``` -Here you will need to add a foreign key on column `posts.user_id`. This ensures +Add a foreign key here on column `posts.user_id`. This ensures that data consistency is enforced on database level. Foreign keys also mean that the database can very quickly remove associated data (for example, when removing a user), instead of Rails having to do this. @@ -28,7 +28,7 @@ Guide](migration_style_guide.md) for more information. Keep in mind that you can only safely add foreign keys to existing tables after you have removed any orphaned rows. The method `add_concurrent_foreign_key` -does not take care of this so you'll need to do so manually. See +does not take care of this so you need to do so manually. See [adding foreign key constraint to an existing column](database/add_foreign_key_to_existing_column.md). ## Cascading Deletes @@ -39,7 +39,7 @@ this should be set to `CASCADE`. ## Indexes When adding a foreign key in PostgreSQL the column is not indexed automatically, -thus you must also add a concurrent index. Not doing so will result in cascading +thus you must also add a concurrent index. Not doing so results in cascading deletes being very slow. ## Naming foreign keys @@ -48,7 +48,7 @@ By default Ruby on Rails uses the `_id` suffix for foreign keys. So we should only use this suffix for associations between two tables. If you want to reference an ID on a third party platform the `_xid` suffix is recommended. -The spec `spec/db/schema_spec.rb` will test if all columns with the `_id` suffix +The spec `spec/db/schema_spec.rb` tests if all columns with the `_id` suffix have a foreign key constraint. So if that spec fails, don't add the column to `IGNORED_FK_COLUMNS`, but instead add the FK constraint, or consider naming it differently. @@ -56,7 +56,7 @@ differently. ## Dependent Removals Don't define options such as `dependent: :destroy` or `dependent: :delete` when -defining an association. Defining these options means Rails will handle the +defining an association. Defining these options means Rails handles the removal of data, instead of letting the database handle this in the most efficient way possible. @@ -80,13 +80,13 @@ foreign keys to remove the data as this would result in the file system data being left behind. In such a case you should use a service class instead that takes care of removing non database data. -In cases where the relation spans multiple databases you will have even +In cases where the relation spans multiple databases you have even further problems using `dependent: :destroy` or the above hooks. You can read more about alternatives at [Avoid `dependent: :nullify` and `dependent: :destroy` across databases](database/multiple_databases.md#avoid-dependent-nullify-and-dependent-destroy-across-databases). -## Alternative primary keys with has_one associations +## Alternative primary keys with `has_one` associations Sometimes a `has_one` association is used to create a one-to-one relationship: @@ -112,9 +112,9 @@ create_table :user_configs, id: false do |t| end ``` -Setting `default: nil` will ensure a primary key sequence is not created, and since the primary key -will automatically get an index, we set `index: false` to avoid creating a duplicate. -You will also need to add the new primary key to the model: +Setting `default: nil` ensures a primary key sequence is not created, and since the primary key +automatically gets an index, we set `index: false` to avoid creating a duplicate. +You also need to add the new primary key to the model: ```ruby class UserConfig < ActiveRecord::Base @@ -126,4 +126,4 @@ end Using a foreign key as primary key saves space but can make [batch counting](service_ping/implement.md#batch-counters) in [Service Ping](service_ping/index.md) less efficient. -Consider using a regular `id` column if the table will be relevant for Service Ping. +Consider using a regular `id` column if the table is relevant for Service Ping. diff --git a/doc/development/gemfile.md b/doc/development/gemfile.md index e0f5a905831..0fcfb88c9cd 100644 --- a/doc/development/gemfile.md +++ b/doc/development/gemfile.md @@ -89,6 +89,7 @@ When upgrading the Rails gem and its dependencies, you also should update the fo You should also update npm packages that follow the current version of Rails: - `@rails/ujs` + - Run `yarn patch-package @rails/ujs` after updating this to ensure our local patch file version matches. - `@rails/actioncable` ## Upgrading dependencies because of vulnerabilities @@ -138,8 +139,8 @@ To avoid upgrading indirect dependencies, we can use [`bundle update When submitting a merge request including a dependency update, include a link to the Gem diff between the 2 versions in the merge request -description. You can find this link on `rubygems.org` under -**Review Changes**. When you click it, RubyGems generates a comparison +description. You can find this link on `rubygems.org`, select +**Review Changes** to generate a comparison between the versions on `diffend.io`. For example, this is the gem diff for [`thor` 1.0.0 vs 1.0.1](https://my.diffend.io/gems/thor/1.0.0/1.0.1). Use the diff --git a/doc/development/geo.md b/doc/development/geo.md index f62b2de30db..18dffe42177 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -33,15 +33,15 @@ for new events and creates background jobs for each specific event type. For example when a repository is updated, the Geo **primary** site creates a Geo event with an associated repository updated event. The Geo Log Cursor daemon -picks the event up and schedules a `Geo::ProjectSyncWorker` job which will -use the `Geo::RepositorySyncService` and `Geo::WikiSyncService` classes +picks the event up and schedules a `Geo::ProjectSyncWorker` job which +uses the `Geo::RepositorySyncService` and `Geo::WikiSyncService` classes to update the repository and the wiki respectively. The Geo Log Cursor daemon can operate in High Availability mode automatically. -The daemon will try to acquire a lock from time to time and once acquired, it -will behave as the *active* daemon. +The daemon tries to acquire a lock from time to time and once acquired, it +behaves as the *active* daemon. -Any additional running daemons on the same site, will be in standby +Any additional running daemons on the same site, is in standby mode, ready to resume work if the *active* daemon releases its lock. We use the [`ExclusiveLease`](https://www.rubydoc.info/github/gitlabhq/gitlabhq/Gitlab/ExclusiveLease) lock type with a small TTL, that is renewed at every @@ -188,16 +188,20 @@ needs to be applied to the tracking database on each **secondary** site. ### Configuration -The database configuration is set in [`config/database_geo.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database_geo.yml.postgresql). +The database configuration is set in [`config/database.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database.yml.postgresql). The directory [`ee/db/geo`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/geo) contains the schema and migrations for this database. -To write a migration for the database, use the `GeoMigrationGenerator`: +To write a migration for the database, run: ```shell -rails g geo_migration [args] [options] +rails g migration [args] [options] --database geo ``` +Geo should continue using `Gitlab::Database::Migration[1.0]` until the `gitlab_geo` schema is supported, and is for the time being exempt from being validated by `Gitlab::Database::Migration[2.0]`. This requires a developer to manually amend the migration file to change from `[2.0]` to `[1.0]` due to the migration defaults being 2.0. + +For more information, see the [Enable Geo migrations to use Migration[2.0]](https://gitlab.com/gitlab-org/gitlab/-/issues/363491) issue. + To migrate the tracking database, run: ```shell diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index 055c2cd4ea8..18774b9b3fd 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -59,7 +59,7 @@ naming conventions: consume) events. It takes care of the communication between the primary site (where events are produced) and the secondary site (where events are consumed). The engineer who wants to incorporate - Geo in their feature will use the API of replicators to make this + Geo in their feature uses the API of replicators to make this happen. - **Geo Domain-Specific Language**: @@ -99,7 +99,7 @@ end The class name should be unique. It also is tightly coupled to the table name for the registry, so for this example the registry table -will be `package_file_registry`. +is `package_file_registry`. For the different data types Geo supports there are different strategies to include. Pick one that fits your needs. diff --git a/doc/development/gitlab_flavored_markdown/specification_guide/index.md b/doc/development/gitlab_flavored_markdown/specification_guide/index.md index 397d555c54f..cedf44cf1fc 100644 --- a/doc/development/gitlab_flavored_markdown/specification_guide/index.md +++ b/doc/development/gitlab_flavored_markdown/specification_guide/index.md @@ -52,7 +52,10 @@ this inconsistency. Some places in the code refer to both the GitLab and GitHub specifications simultaneous in the same areas of logic. In these situations, _GitHub_ Flavored Markdown may be referred to with variable or constant names like -`ghfm_` to avoid confusion. +`ghfm_` to avoid confusion. For example, we use the `ghfm` acronym for the +[`ghfm_spec_v_0.29.txt` GitHub Flavored Markdown specification file](#github-flavored-markdown-specification) +which is committed to the `gitlab` repo and used as input to the +[`update_specification.rb` script](#update-specificationrb-script). The original CommonMark specification is referred to as _CommonMark_ (no acronym). @@ -141,6 +144,8 @@ and the existing GLFM parser and render implementations. They may also be manually updated as necessary to test-drive incomplete implementations. Regarding the terminology used here: +<!-- vale gitlab.InclusionCultural = NO --> + 1. The Markdown snapshot tests can be considered a form of the [Golden Master Testing approach](https://www.google.com/search?q=golden+master+testing), which is also referred to as Approval Testing or Characterization Testing. @@ -167,6 +172,11 @@ Regarding the terminology used here: they are colocated under the `spec/fixtures` directory with the rest of the fixture data for the GitLab Rails application. +<!-- vale gitlab.InclusionCultural = YES --> + +See also the section on [normalization](#normalization) below, which is an important concept used +in the Markdown snapshot testing. + ## Parsing and Rendering The Markdown dialect used in the GitLab application has a dual requirement for rendering: @@ -187,7 +197,7 @@ implementations: It leverages the [`commonmarker`](https://github.com/gjtorikian/commonmarker) gem, which is a Ruby wrapper for [`libcmark-gfm`](https://github.com/github/cmark), GitHub's fork of the reference parser for CommonMark. `libcmark-gfm` is an extended - version of the C reference implementation of [CommonMark](http://commonmark.org/) + version of the C reference implementation of [CommonMark](https://commonmark.org/) 1. The frontend parser / renderer supports parsing and _WYSIWYG_ rendering for the Content Editor. It is implemented in JavaScript. Parsing is based on the [Remark](https://github.com/remarkjs/remark) Markdown parser, which produces a @@ -213,32 +223,42 @@ HTML which differs from the canonical HTML examples from the specification. For every Markdown example in the GLFM specification, three versions of HTML can potentially be rendered from the example: -1. **Static HTML**: HTML produced by the backend (Ruby) renderer, which - contains extra styling and behavioral HTML. For example, **Create task** buttons - added for dynamically creating an issue from a task list item. - The GitLab [Markdown API](../../../api/markdown.md) generates HTML - for a given Markdown string using this method. -1. **WYSIWYG HTML**: HTML produced by the frontend (JavaScript) Content Editor, - which includes parsing and rendering logic. Used to present an editable document - in the ProseMirror WYSIWYG editor. -1. **Canonical HTML**: The clean, basic version of HTML rendered from Markdown. - 1. For the examples which come from the CommonMark specification and - GFM extensions specification, - the canonical HTML is the exact identical HTML found in the - GFM - `spec.txt` example blocks. - 1. For GLFM extensions to the <abbr title="GitHub Flavored Markdown">GFM</abbr> / CommonMark - specification, a `glfm_canonical_examples.txt` - [input specification file](#input-specification-files) contains the - Markdown examples and corresponding canonical HTML examples. +- Static HTML. +- WYSIWYG HTML. +- Canonical HTML. + +#### Static HTML + +**Static HTML** is HTML produced by the backend (Ruby) renderer, which +contains extra styling and behavioral HTML. For example, **Create task** buttons +added for dynamically creating an issue from a task list item. +The GitLab [Markdown API](../../../api/markdown.md) generates HTML +for a given Markdown string using this method. + +#### WYSIWYG HTML + +**WYSIWYG HTML** is HTML produced by the frontend (JavaScript) Content Editor, +which includes parsing and rendering logic. It is used to present an editable document +in the ProseMirror WYSIWYG editor. -As the rendered static and WYSIWYG HTML from the backend (Ruby) and frontend (JavaScript) -renderers contain extra HTML, their rendered HTML can be converted to canonical HTML -by a [canonicalization](#canonicalization-of-html) process. +#### Canonical HTML -#### Canonicalization of HTML +**Canonical HTML** is the clean, basic version of HTML rendered from Markdown. -Neither the backend (Ruby) nor the frontend (JavaScript) rendered can directly render canonical HTML. +1. For the examples which come from the CommonMark specification and + GFM extensions specification, the canonical HTML is the exact identical HTML found in the + GFM `spec.txt` example blocks. +1. For GLFM extensions to the <abbr title="GitHub Flavored Markdown">GFM</abbr> / CommonMark + specification, a `glfm_canonical_examples.txt` [input specification file](#input-specification-files) + contains the Markdown examples and corresponding canonical HTML examples. + +### Canonicalization of HTML + +The rendered [static HTML](#static-html) and [WYSIWYG HTML](#wysiwyg-html) +from the backend (Ruby) and frontend (JavaScript) renderers usually contains extra styling +or HTML elements, to support specific appearance and behavioral requirements. + +Neither the backend nor the frontend rendering logic can directly render the clean, basic canonical HTML. Nor should they be able to, because: - It's not a direct requirement to support any GitLab application feature. @@ -258,6 +278,49 @@ HTML. (For example, when they are represented as an image.) In these cases, the conformance test for the example can be skipped by setting `skip_update_example_snapshots: true` for the example in `glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml`. +### Normalization + +Versions of the rendered HTML and ProseMirror JSON can vary for a number of reasons. +Differences in styling or HTML structure can occur, but the values of attributes or nodes may +also vary across different test runs or environments. For example: + +1. Database record identifiers +1. Namespace or project identifiers +1. Portions of URIs +1. File paths or names +1. Random values + +For the [Markdown snapshot testing](#markdown-snapshot-testing) to work +properly, you must account for these differences in a way that ensures the tests are reliable, +and always behave the same across different test runs or environments. + +To account for these differences, there is a process called **_normalization_**. Normalization +allows custom regular expressions with +[_capturing groups_](https://ruby-doc.org/core-3.1.2/Regexp.html#class-Regexp-label-Capturing) +to be applied to two different versions of HTML or JSON for a given Markdown example, +and the contents of the captured groups can be replaced with the same fixed values. + +Then, the two normalized versions can be compared to each other to ensure all other non-variable +content is identical. + +NOTE: +We don't care about verifying specific attribute values here, so +it's OK if the normalizations discard and replace these variable values with fixed values. +Different testing levels have different purposes: + +1. [Markdown snapshot testing](#markdown-snapshot-testing) is intended to enforce the structure of + the rendered HTML/JSON, and to ensure that it conforms to the canonical specification. +1. Individual unit tests of the implementation for a specific Markdown example are responsible for + specific and targeted testing of these variable values. + +We also use this same regex capture-and-replace normalization approach for +[canonicalization of HTML](#canonicalization-of-html), because it is essentially the same process. +With canonicalization, instead of just replacing variable values, we are removing non-canonical +portions of the HTML. + +Refer to [`glfm_example_normalizations.yml`](#glfm_example_normalizationsyml) for a detailed explanation +of how the normalizations are specified. + ## Goals Given the constraints above, we have a few goals related to the GLFM @@ -374,7 +437,7 @@ subgraph script: A --> B{Backend Markdown API} end subgraph input:<br/>input specification files - C[gfm_spec_v_0.29.txt] --> A + C[ghfm_spec_v_0.29.txt] --> A D[glfm_intro.txt] --> A E[glfm_canonical_examples.txt] --> A end @@ -512,12 +575,16 @@ updated, as in the case of all GFM files. ##### GitHub Flavored Markdown specification -[`glfm_specification/input/github_flavored_markdown/gfm_spec_v_0.29.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/github_flavored_markdown/gfm_spec_v_0.29.txt) +[`glfm_specification/input/github_flavored_markdown/ghfm_spec_v_0.29.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/github_flavored_markdown/ghfm_spec_v_0.29.txt) is the official latest [GFM `spec.txt`](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). - It is automatically downloaded and updated by `update-specification.rb` script. - When it is downloaded, the version number is added to the filename. +NOTE: +This file uses the `ghfm` acronym instead of `gfm`, as +explained in the [Acronyms section](#acronyms-glfm-ghfm-gfm-commonmark). + ##### `glfm_intro.txt` [`glfm_specification/input/gitlab_flavored_markdown/glfm_intro.txt`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/gitlab_flavored_markdown/glfm_intro.txt) @@ -578,23 +645,114 @@ The actual file should not have these prefixed `|` characters. controls the behavior of the [scripts](#scripts) and [tests](#types-of-markdown-tests-driven-by-the-glfm-specification). - It is manually updated. -- It controls the status of automatic generation of files based on Markdown examples. -- It allows example snapshot generation, Markdown conformance tests, or - Markdown snapshot tests to be skipped for individual examples. For example, if - they are unimplemented, broken, or cannot be tested for some reason. +- The `skip_update_example_snapshot*` fields control the status of automatic generation of + snapshot example entries based on Markdown examples. +- The `skip_running_*` control allow Markdown conformance tests or + Markdown snapshot tests to be skipped for individual examples. +- This allows control over skipping this processing or testing of various examples when they + are unimplemented, partially implemented, broken, cannot be generated, or cannot be tested for some reason. +- All entries default to false. They can be set to true by specifying a Ruby + value which evaluates as truthy. This could be the boolean `true` value, but ideally should + be a string describing why the example's updating or testing is being skipped. +- When a `skip_update_example_snapshot*` entry is true, the existing value is preserved. + However, since the YAML is re-written, the style of the string value and its + [Block Chomping Indicator (`|`)](https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator) + may be modified, because the Ruby `psych` YAML library automatically determines this. + +The following optional entries are supported for each example. They all default to `false`: + +- `skip_update_example_snapshots`: When true, skips any addition or update of any this example's entries + in the [`spec/fixtures/glfm/example_snapshots/html.yml`](#specfixturesglfmexample_snapshotshtmlyml) file + or the [`spec/fixtures/glfm/example_snapshots/prosemirror_json.yml`](#specfixturesglfmexample_snapshotsprosemirror_jsonyml) file. + If this value is truthy, then no other `skip_update_example_snapshot_*` entries can be truthy, + and an error is raised if any of them are. +- `skip_update_example_snapshot_html_static`: When true, skips addition or update of this example's [static HTML](#static-html) + entry in the [`spec/fixtures/glfm/example_snapshots/html.yml`](#specfixturesglfmexample_snapshotshtmlyml) file. +- `skip_update_example_snapshot_html_wysiwyg`: When true, skips addition or update of this example's [WYSIWYG HTML](#wysiwyg-html) + entry in the [`spec/fixtures/glfm/example_snapshots/html.yml`](#specfixturesglfmexample_snapshotshtmlyml) file. +- `skip_update_example_snapshot_prosemirror_json`: When true, skips addition or update of this example's + entry in the [`spec/fixtures/glfm/example_snapshots/prosemirror_json.yml`](#specfixturesglfmexample_snapshotsprosemirror_jsonyml) file. +- `skip_running_conformance_static_tests`: When true, skips running the [Markdown conformance tests](#markdown-conformance-testing) + of the [static HTML](#static-html) for this example. +- `skip_running_conformance_wysiwyg_tests`: When true, skips running the [Markdown conformance tests](#markdown-conformance-testing) + of the [WYSIWYG HTML](#wysiwyg-html) for this example. +- `skip_running_snapshot_static_html_tests`: When true, skips running the [Markdown snapshot tests](#markdown-snapshot-testing) + of the [static HTML](#multiple-versions-of-rendered-html) for this example. +- `skip_running_snapshot_wysiwyg_html_tests`: When true, skips running the [Markdown snapshot tests](#markdown-snapshot-testing) + of the [WYSIWYG HTML](#wysiwyg-html) for this example. +- `skip_running_snapshot_prosemirror_json_tests`: When true, skips running the [Markdown snapshot tests](#markdown-snapshot-testing) + of the [ProseMirror JSON](#specfixturesglfmexample_snapshotsprosemirror_jsonyml) for this example. `glfm_specification/input/gitlab_flavored_markdown/glfm_example_status.yml` sample entry: ```yaml 07_99_an_example_with_incomplete_wysiwyg_implementation_1: - skip_update_example_snapshots: false - skip_update_example_snapshot_html_static: false - skip_update_example_snapshot_html_wysiwyg: false - skip_running_conformance_static_tests: false - skip_running_conformance_wysiwyg_tests: false - skip_running_snapshot_static_html_tests: false - skip_running_snapshot_wysiwyg_html_tests: false - skip_running_snapshot_prosemirror_json_tests: false + skip_update_example_snapshots: 'An explanation of the reason for skipping.' + skip_update_example_snapshot_html_static: 'An explanation of the reason for skipping.' + skip_update_example_snapshot_html_wysiwyg: 'An explanation of the reason for skipping.' + skip_update_example_snapshot_prosemirror_json: 'An explanation of the reason for skipping.' + skip_running_conformance_static_tests: 'An explanation of the reason for skipping.' + skip_running_conformance_wysiwyg_tests: 'An explanation of the reason for skipping.' + skip_running_snapshot_static_html_tests: 'An explanation of the reason for skipping.' + skip_running_snapshot_wysiwyg_html_tests: 'An explanation of the reason for skipping.' + skip_running_snapshot_prosemirror_json_tests: 'An explanation of the reason for skipping.' +``` + +##### `glfm_example_normalizations.yml` + +[`glfm_specification/input/gitlab_flavored_markdown/glfm_example_normalizations.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/glfm_specification/input/gitlab_flavored_markdown/glfm_example_normalizations.yml) +controls the [normalization](#normalization) process. It allows one or more `regex`/`replacement` pairs +to be specified for a Markdown example. + +- It is manually updated. +- It has a nested structure corresponding to the example and type of entry it refers to. +- It extensively uses [YAML anchors and aliases](https://yaml.org/spec/1.2.2/#692-node-anchors) + to avoid duplication of `regex`/`replacement` pairs and allow them to be shared across multiple examples. +- The YAML anchors use a naming convention based on the index number of the example, to + ensure unique anchor names and avoid naming conflicts. + +`glfm_specification/input/gitlab_flavored_markdown/glfm_example_normalizations.yml` sample entries: + +```yaml +# NOTE: All YAML anchors which are shared across one or more examples are defined in the `00_shared` section. +00_shared: + 00_uri: &00_uri + - regex: '(href|data-src)(=")(.*?)(test-file\.(png|zip)")' + replacement: '\1\2URI_PREFIX\4' +01_01__section_one__example_containing_a_uri__001: + html: + static: + canonical: + 01_01_uri: *00_uri + snapshot: + 01_01_uri: *00_uri + wysiwyg: + 01_01_uri: *00_uri + prosemirror_json: + 01_01_uri: *00_uri +07_01__gitlab_specific_markdown__footnotes__001: + # YAML anchors which are only shared within a single example should be defined within the example + shared: + 07_01_href: &07_01_href + - regex: '(href)(=")(.+?)(")' + replacement: '\1\2REF\4' + 07_01_id: &07_01_id + - regex: '(id)(=")(.+?)(")' + replacement: '\1\2ID\4' + html: + static: + canonical: + 07_01_href: *07_01_href + 07_01_id: *07_01_id + snapshot: + 07_01_href: *07_01_href + 07_01_id: *07_01_id + wysiwyg: + 07_01_href: *07_01_href + 07_01_id: *07_01_id + prosemirror_json: + 07_01_href: *07_01_href + 07_01_id: *07_01_id ``` #### Output specification files @@ -610,7 +768,8 @@ are colocated under the same parent folder `glfm_specification` with the other a mix of manually edited and generated files. In GFM, `spec.txt` is [located in the test dir](https://github.com/github/cmark-gfm/blob/master/test/spec.txt), -and in CommonMark it's located [in the project root](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). No precedent exists for a standard location. In the future, we may decide to +and in CommonMark it's located [in the project root](https://github.com/github/cmark-gfm/blob/master/test/spec.txt). +No precedent exists for a standard location. In the future, we may decide to move or copy a hosted version of the rendered HTML `spec.html` version to another location or site. ##### spec.txt @@ -748,12 +907,12 @@ Any exceptions or failures which occur when generating HTML are replaced with an ```yaml 06_04_inlines_emphasis_and_strong_emphasis_1: - canonical: | - <p><em>foo bar</em></p> - static: | - <p data-sourcepos="1:1-1:9" dir="auto"><strong>foo bar</strong></p> - wysiwyg: | - <p><strong>foo bar</strong></p> + canonical: | + <p><em>foo bar</em></p> + static: | + <p data-sourcepos="1:1-1:9" dir="auto"><strong>foo bar</strong></p> + wysiwyg: | + <p><strong>foo bar</strong></p> ``` NOTE: diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index a5661a77da3..f5b0da2f162 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -88,7 +88,7 @@ of the "GitLab" project on the Engineering Projects page in the handbook. To add yourself to this list, add the following to your profile in the -[team.yml](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/team.yml) +[`team.yml`](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/team.yml) file and ask your manager to review and merge. ```yaml @@ -406,9 +406,8 @@ variable). Since daemons are long-running applications, they should have mechanisms to manage cancellations, and avoid unnecessary resources consumption (which could -lead to DDOS vulnerabilities). [Go -Context](https://github.com/golang/go/wiki/CodeReviewComments#contexts) should -be used in functions that can block and passed as the first parameter. +lead to DDoS vulnerabilities). [Go Context](https://github.com/golang/go/wiki/CodeReviewComments#contexts) +should be used in functions that can block and passed as the first parameter. ## Dockerfiles diff --git a/doc/development/graphql_guide/batchloader.md b/doc/development/graphql_guide/batchloader.md index 0e90f89ff7a..492d3bc9007 100644 --- a/doc/development/graphql_guide/batchloader.md +++ b/doc/development/graphql_guide/batchloader.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -12,7 +12,7 @@ It is the properties of the GraphQL query tree that create opportunities for bat ## When should you use it? -We should try to batch DB requests as much as possible during GraphQL **query** execution. There is no need to batch loading during **mutations** because they are executed serially. If you need to make a database query, and it is possible to combine two similar (but not identical) queries, then consider using the batch-loader. +We should try to batch DB requests as much as possible during GraphQL **query** execution. There is no need to batch loading during **mutations** because they are executed serially. If you need to make a database query, and it is possible to combine two similar (but not necessarily identical) queries, then consider using the batch-loader. When implementing a new endpoint we should aim to minimise the number of SQL queries. For stability and scalability we must also ensure that our queries do not suffer from N+1 performance issues. @@ -20,7 +20,7 @@ When implementing a new endpoint we should aim to minimise the number of SQL que Batch loading is useful when a series of queries for inputs `Qα, Qβ, ... Qω` can be combined to a single query for `Q[α, β, ... ω]`. An example of this is lookups by ID, where we can find two users by usernames as cheaply as one, but real-world examples can be more complex. -Batch loading is not suitable when the result sets have different sort-orders, grouping, aggregation or other non-composable features. +Batch loading is not suitable when the result sets have different sort orders, grouping, aggregation, or other non-composable features. There are two ways to use the batch-loader in your code. For simple ID lookups, use `::Gitlab::Graphql::Loaders::BatchModelLoader.new(model, id).find`. For more complex cases, you can use the batch API directly. @@ -47,9 +47,29 @@ end Here an [example MR](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46549) illustrating how to use our `BatchLoading` mechanism. +## The `BatchModelLoader` + +For ID lookups, the advice is to use the `BatchModelLoader`: + +```ruby +def project + ::Gitlab::Graphql::Loaders::BatchModelLoader.new(::Project, object.project_id).find +end +``` + +To preload associations, you can pass an array of them: + +```ruby +def issue(lookahead:) + preloads = [:author] if lookahead.selects?(:author) + + ::Gitlab::Graphql::Loaders::BatchModelLoader.new(::Issue, object.issue_id, preloads).find +end +``` + ## How does it work exactly? -Each lazy object knows which data it needs to load and how to batch the query. When we need to use the lazy objects (which we announce by calling `#sync`), they will be loaded along with all other similar objects in the current batch. +Each lazy object knows which data it needs to load and how to batch the query. When we need to use the lazy objects (which we announce by calling `#sync`), they are loaded along with all other similar objects in the current batch. Inside the block we execute a batch query for our items (`User`). After that, all we have to do is to call loader by passing an item which was used in `BatchLoader::GraphQL.for` method (`usernames`) and the loaded object itself (`user`): @@ -61,9 +81,28 @@ BatchLoader::GraphQL.for(username).batch do |usernames, loader| end ``` +The batch-loader uses the source code location of the block to determine +which requests belong in the same queue, but only one instance of the block +is evaluated for each batch. You do not control which one. + +For this reason, it is important that: + +- The block must not refer to (close over) any instance state on objects. The best practice + is to pass all data the block needs through to it in the `for(data)` call. +- The block must be specific to a kind of batched data. Implementing generic + loaders (such as the `BatchModelLoader`) is possible, but it requires the use + of an injective `key` argument. +- Batches are not shared unless they refer to the same block - two identical blocks + with the same behavior, parameters, and keys do not get shared. For this reason, + never implement batched ID lookups on your own, instead use the `BatchModelLoader` for + maximum sharing. If you see two fields define the same batch-loading, consider + extracting that out to a new `Loader`, and enabling them to share. + ### What does lazy mean? -It is important to avoid syncing batches too early. In the example below we can see how calling sync too early can eliminate opportunities for batching: +It is important to avoid syncing batches (forcing their evaluation) too early. The following example shows how calling sync too early can eliminate opportunities for batching. + +This example calls sync on `x` too early: ```ruby x = find_lazy(1) @@ -80,6 +119,8 @@ z.sync # => will run 2 queries ``` +However, this example waits until all requests are queued, and eliminates the extra query: + ```ruby x = find_lazy(1) y = find_lazy(2) @@ -92,9 +133,38 @@ z.sync # => will run 1 query ``` +NOTE: +There is no dependency analysis in the use of batch-loading. There is simply +a pending queue of requests, and as soon as any one result is needed, all pending +requests are evaluated. + +You should never call `batch.sync` or use `Lazy.force` in resolver code. +If you depend on a lazy value, use `Lazy.with_value` instead: + +```ruby +def publisher + ::Gitlab::Graphql::Loaders::BatchModelLoader.new(::Publisher, object.publisher_id).find +end + +# Here we need the publisher in order to generate the catalog URL +def catalog_url + ::Gitlab::Graphql::Lazy.with_value(publisher) do |p| + UrlHelpers.book_catalog_url(publisher, object.isbn) + end +end +``` + ## Testing -Any GraphQL field that supports `BatchLoading` should be tested using the `batch_sync` method available in [GraphQLHelpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/helpers/graphql_helpers.rb). +Ideally, do all your testing using request specs, and using `Schema.execute`. If +you do so, you do not need to manage the lifecycle of lazy values yourself, and +you are assured accurate results. + +GraphQL fields that return lazy values may need these values forced in tests. +Forcing refers to explicit demands for evaluation, where this would normally +be arranged by the framework. + +You can force a lazy value with the `GraphqlHelpers#batch_sync` method available in [GraphQLHelpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/helpers/graphql_helpers.rb), or by using `Gitlab::Graphql::Lazy.force`. For example: ```ruby it 'returns data as a batch' do @@ -114,8 +184,8 @@ We can also use [QueryRecorder](../query_recorder.md) to make sure we are perfor ```ruby it 'executes only 1 SQL query' do - query_count = ActiveRecord::QueryRecorder.new { subject }.count + query_count = ActiveRecord::QueryRecorder.new { subject } - expect(query_count).to eq(1) + expect(query_count).not_to exceed_query_limit(1) end ``` diff --git a/doc/development/graphql_guide/pagination.md b/doc/development/graphql_guide/pagination.md index 1f40a605cfe..72f321a1fd2 100644 --- a/doc/development/graphql_guide/pagination.md +++ b/doc/development/graphql_guide/pagination.md @@ -23,7 +23,7 @@ and used across much of GitLab. You can recognize it by a list of page numbers near the bottom of a page, which, when clicked, take you to that page of results. -For example, when you click **Page 100**, we send `100` to the +For example, when you select **Page 100**, we send `100` to the backend. For example, if each page has say 20 items, the backend calculates `20 * 100 = 2000`, and it queries the database by offsetting (skipping) the first 2000 diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md index 881369e429b..731639b6f06 100644 --- a/doc/development/hash_indexes.md +++ b/doc/development/hash_indexes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index afc04045763..8231cf4316b 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -60,6 +60,7 @@ are very appreciative of the work done by translators and proofreaders! - German - Michael Hahnle - [GitLab](https://gitlab.com/mhah), [Crowdin](https://crowdin.com/profile/mhah) - Katrin Leinweber - [GitLab](https://gitlab.com/katrinleinweber), [Crowdin](https://crowdin.com/profile/katrinleinweber) + - Justman10000 - [GitLab](https://gitlab.com/Justman10000), [Crowdin](https://crowdin.com/profile/Justman10000) - Greek - Proofreaders needed. - Hebrew @@ -101,7 +102,7 @@ are very appreciative of the work done by translators and proofreaders! - Horberlan Brito - [GitLab](https://gitlab.com/horberlan), [Crowdin](https://crowdin.com/profile/horberlan) - Romanian - Mircea Pop - [GitLab](https://gitlab.com/eeex), [Crowdin](https://crowdin.com/profile/eex) - - Rareș Pița - [GitLab](https://gitlab.com/dlphin), [Crowdin](https://crowdin.com/profile/dlphin) + - Rareș Pița - [GitLab](https://gitlab.com/dlphin) - Nicolae Liviu - [GitLab](https://gitlab.com/nicklcanada), [Crowdin](https://crowdin.com/profile/nicklcanada) - Russian - Nikita Grylov - [GitLab](https://gitlab.com/nixel2007), [Crowdin](https://crowdin.com/profile/nixel2007) @@ -147,7 +148,7 @@ translations to the GitLab project. 1. Request proofreader permissions by opening a merge request to add yourself to the list of proofreaders. - Open the [`proofreader.md` source file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/i18n/proofreader.md) and click **Edit**. + Open the [`proofreader.md` source file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/i18n/proofreader.md) and select **Edit**. Add your language in alphabetical order and add yourself to the list, including: diff --git a/doc/development/image_scaling.md b/doc/development/image_scaling.md index e1ffbdb766a..93575429369 100644 --- a/doc/development/image_scaling.md +++ b/doc/development/image_scaling.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Memory info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/insert_into_tables_in_batches.md b/doc/development/insert_into_tables_in_batches.md index cd659a3d19b..ebed3d16319 100644 --- a/doc/development/insert_into_tables_in_batches.md +++ b/doc/development/insert_into_tables_in_batches.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments description: "Sometimes it is necessary to store large amounts of records at once, which can be inefficient @@ -48,7 +48,7 @@ records = [MyModel.new, ...] MyModel.bulk_insert!(records) ``` -Note that calls to `bulk_insert!` will always attempt to insert _new records_. If instead +Calls to `bulk_insert!` always attempt to insert _new records_. If instead you would like to replace existing records with new values, while still inserting those that do not already exist, then you can use `bulk_upsert!`: @@ -59,9 +59,9 @@ MyModel.bulk_upsert!(records, unique_by: [:name]) ``` In this example, `unique_by` specifies the columns by which records are considered to be -unique and as such will be updated if they existed prior to insertion. For example, if +unique and as such are updated if they existed prior to insertion. For example, if `existing_model` has a `name` attribute, and if a record with the same `name` value already -exists, its fields will be updated with those of `existing_model`. +exists, its fields are updated with those of `existing_model`. The `unique_by` parameter can also be passed as a `Symbol`, in which case it specifies a database index by which a column is considered unique: @@ -72,8 +72,8 @@ MyModel.bulk_insert!(records, unique_by: :index_on_name) ### Record validation -The `bulk_insert!` method guarantees that `records` will be inserted transactionally, and -will run validations on each record prior to insertion. If any record fails to validate, +The `bulk_insert!` method guarantees that `records` are inserted transactionally, and +runs validations on each record prior to insertion. If any record fails to validate, an error is raised and the transaction is rolled back. You can turn off validations via the `:validate` option: @@ -83,7 +83,7 @@ MyModel.bulk_insert!(records, validate: false) ### Batch size configuration -In those cases where the number of `records` is above a given threshold, insertions will +In those cases where the number of `records` is above a given threshold, insertions occur in multiple batches. The default batch size is defined in [`BulkInsertSafe::DEFAULT_BATCH_SIZE`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). Assuming a default threshold of 500, inserting 950 records @@ -95,7 +95,7 @@ MyModel.bulk_insert!(records, batch_size: 100) ``` Assuming the same number of 950 records, this would result in 10 batches being written instead. -Since this will also affect the number of `INSERT`s that occur, make sure you measure the +Since this also affects the number of `INSERT` statements that occur, make sure you measure the performance impact this might have on your code. There is a trade-off between the number of `INSERT` statements the database has to process and the size and cost of each `INSERT`. @@ -127,7 +127,7 @@ records are inserted in bulk, we currently prevent their use. The specifics around which callbacks are explicitly allowed are defined in [`BulkInsertSafe`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). Consult the module source code for details. If your class uses callbacks that are not explicitly designated -safe and you `include BulkInsertSafe` the application will fail with an error. +safe and you `include BulkInsertSafe` the application fails with an error. ### `BulkInsertSafe` versus `InsertAll` @@ -155,7 +155,7 @@ owner = OwnerModel.new(owned_relations: array_of_owned_relations) owner.save! ``` -This will issue a single `INSERT`, and transaction, for every record in `owned_relations`, which is inefficient if +This issues a single `INSERT`, and transaction, for every record in `owned_relations`, which is inefficient if `array_of_owned_relations` is large. To remedy this, the `BulkInsertableAssociations` concern can be used to declare that the owner defines associations that are safe for bulk insertion: @@ -180,8 +180,8 @@ BulkInsertableAssociations.with_bulk_insert do end ``` -Note that you can still save relations that are not `BulkInsertSafe` in this block; they will -simply be treated as if you had invoked `save` from outside the block. +You can still save relations that are not `BulkInsertSafe` in this block; they +simply are treated as if you had invoked `save` from outside the block. ## Known limitations @@ -192,5 +192,5 @@ There are a few restrictions to how these APIs can be used: - It does not yet support `has_many through: ...` relations. Moreover, input data should either be limited to around 1000 records at most, -or already batched prior to calling bulk insert. The `INSERT` statement will run in a single +or already batched prior to calling bulk insert. The `INSERT` statement runs in a single transaction, so for large amounts of records it may negatively affect database stability. diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index e595fea6d96..604e481a809 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -43,7 +43,7 @@ if you need clarification or spot any outdated information. ### Define properties Integrations can define arbitrary properties to store their configuration with the class method `Integration.prop_accessor`. -The values are stored as a serialized JSON hash in the `integrations.properties` column. +The values are stored as an encrypted JSON hash in the `integrations.encrypted_properties` column. For example: diff --git a/doc/development/integrations/jenkins.md b/doc/development/integrations/jenkins.md index 8a3f64f0a0d..f430fc380b1 100644 --- a/doc/development/integrations/jenkins.md +++ b/doc/development/integrations/jenkins.md @@ -36,8 +36,8 @@ GitLab does not allow requests to localhost or the local network by default. Whe Jenkins uses the GitLab API and needs an access token. 1. Sign in to your GitLab instance. -1. Click on your profile picture, then click **Settings**. -1. Click **Access Tokens**. +1. Select your profile picture, then select **Settings**. +1. Select **Access Tokens**. 1. Create a new Access Token with the **API** scope enabled. Note the value of the token. ## Configure Jenkins diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index 26ef67c937c..ade81e29ffb 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -37,13 +37,13 @@ To install the app in Jira: Marketplace: 1. In Jira, navigate to **Jira settings > Apps > Manage apps**. - 1. Scroll to the bottom of the **Manage apps** page and click **Settings**. - 1. Select **Enable development mode** and click **Apply**. + 1. Scroll to the bottom of the **Manage apps** page and select **Settings**. + 1. Select **Enable development mode** and select **Apply**. 1. Install the app: 1. In Jira, navigate to **Jira settings > Apps > Manage apps**. - 1. Click **Upload app**. + 1. Select **Upload app**. 1. In the **From this URL** field, provide a link to the app descriptor. The host and port must point to your GitLab instance. For example: @@ -52,10 +52,10 @@ To install the app in Jira: https://xxxx.gitpod.io/-/jira_connect/app_descriptor.json ``` - 1. Click **Upload**. + 1. Select **Upload**. If the install was successful, you should see the **GitLab.com for Jira Cloud** app under **Manage apps**. - You can also click **Getting Started** to open the configuration page rendered from your GitLab instance. + You can also select **Getting Started** to open the configuration page rendered from your GitLab instance. _Note that any changes to the app descriptor requires you to uninstall then reinstall the app._ @@ -106,11 +106,7 @@ The following steps describe setting up an environment to test the GitLab OAuth - Trusted: **No** - Confidential: **No** 1. Copy the Application ID. +1. Go to **Admin > Settings > General**. +1. Scroll down and expand the GitLab for Jira App section. 1. Go to [gitpod.io/variables](https://gitpod.io/variables). -1. Create a new variable named `JIRA_CONNECT_OAUTH_CLIENT_ID`, with a scope of `*/*`, and paste the Application ID as the value. - -If you already have an active Gitpod instance, use the following command in the Gitpod terminal to set the environment variable: - -```shell -eval $(gp env -e JIRA_CONNECT_OAUTH_CLIENT_ID=$YOUR_APPLICATION_ID) -``` +1. Paste the Application ID into the **Jira Connect Application ID** field and click **Save changes** diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 0f4fa1a97a8..1a51ee88c58 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -312,8 +312,7 @@ The format is extensively described in the documentation of [SAST](../../user/application_security/sast/index.md#reports-json-format), [DAST](../../user/application_security/dast/#reports), [Dependency Scanning](../../user/application_security/dependency_scanning/index.md#reports-json-format), -[Container Scanning](../../user/application_security/container_scanning/index.md#reports-json-format), -and [Cluster Image Scanning](../../user/application_security/cluster_image_scanning/index.md#reports-json-format). +and [Container Scanning](../../user/application_security/container_scanning/index.md#reports-json-format) You can find the schemas for these scanners here: @@ -333,33 +332,16 @@ GitLab has the following retention policies for vulnerabilities on non-default b To view vulnerabilities, either: -- Re-run the pipeline. +- Run a new pipeline. - Download the related CI job artifacts if they are available. NOTE: This does not apply for the vulnerabilities existing on the default branch. -### Enable report validation - -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/354928) in GitLab 14.9, and planned for removal in GitLab 15.0. -DISCLAIMER: -This page contains information related to upcoming products, features, and functionality. -It is important to note that the information presented is for informational purposes only. -Please do not rely on this information for purchasing or planning purposes. -As with all projects, the items mentioned on this page are subject to change or delay. -The development, release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. -In GitLab 15.0 and later, report validation is enabled and enforced. Reports that fail validation -are not ingested, and an error message displays on the corresponding pipeline. - -In GitLab 14.10 and later, report validation against the schemas is enabled but not enforced. -Reports that fail validation are ingested but display a warning in the pipeline security tab. - -To enforce report validation for GitLab version 14.10 and earlier, set -[`VALIDATE_SCHEMA`](../../user/application_security/#enable-security-report-validation) to `"true"`. - ### Report validation +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351000) in GitLab 15.0. + You must ensure that reports generated by the scanner pass validation against the schema version declared in your reports. Reports that don't pass validation are not ingested by GitLab, and an error message displays on the corresponding pipeline. diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index 34e0aaedfaf..63f86a3f95d 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -43,7 +43,7 @@ best place to integrate your own product and its results into GitLab. implications for app security, corporate policy, or compliance. When complete, the job reports back on its status and creates a [job artifact](../../ci/pipelines/job_artifacts.md) as a result. -- The [Merge Request Security Widget](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports) +- The [Merge Request Security Widget](../../ci/testing/index.md#security-reports) displays the results of the pipeline's security checks and the developer can review them. The developer can review both a summary and a detailed version of the results. @@ -90,12 +90,11 @@ and complete an integration with the Secure stage. - Documentation for [SAST reports](../../user/application_security/sast/index.md#reports-json-format). - Documentation for [Dependency Scanning reports](../../user/application_security/dependency_scanning/index.md#reports-json-format). - Documentation for [Container Scanning reports](../../user/application_security/container_scanning/index.md#reports-json-format). - - Documentation for [`cluster_image_scanning` reports](../../user/application_security/cluster_image_scanning/index.md#reports-json-format). - See this [example secure job definition that also defines the artifact created](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml). - If you need a new kind of scan or report, [create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new#) and add the label `devops::secure`. - Once the job is completed, the data can be seen: - - In the [Merge Request Security Report](../../user/project/merge_requests/testing_and_reports_in_merge_requests.md#security-reports) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). + - In the [Merge Request Security Report](../../ci/testing/index.md#security-reports) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). - While [browsing a Job Artifact](../../ci/pipelines/job_artifacts.md). - In the [Security Dashboard](../../user/application_security/security_dashboard/index.md) ([Dashboard data flow](https://gitlab.com/snippets/1910005#project-and-group-dashboards)). 1. Optional: Provide a way to interact with results as Vulnerabilities: diff --git a/doc/development/internal_api/index.md b/doc/development/internal_api/index.md index dca71413564..288c0056821 100644 --- a/doc/development/internal_api/index.md +++ b/doc/development/internal_api/index.md @@ -254,7 +254,7 @@ recovery codes based on their SSH key. | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| | `key_id` | integer | no | The ID of the SSH key used as found in the authorized-keys file or through the `/authorized_keys` check | -| `user_id` | integer | no | **Deprecated** User_id for which to generate new recovery codes | +| `user_id` | integer | no | **Deprecated** User ID for which to generate new recovery codes | ```plaintext GET /internal/two_factor_recovery_codes @@ -331,6 +331,37 @@ Example response: - GitLab Shell +## Authenticate Error Tracking requests + +This endpoint is called by the error tracking Go REST API application to authenticate a project. + +| Attribute | Type | Required | Description | +|:-------------|:--------|:---------|:-------------------------------------------------------------------| +| `project_id` | integer | yes | The ID of the project which has the associated key. | +| `public_key` | string | yes | The public key generated by the integrated error tracking feature. | + +```plaintext +POST /internal/error_tracking_allowed +``` + +Example request: + +```shell +curl --request POST --header "Gitlab-Shared-Secret: <Base64 encoded secret>" \ + --data "project_id=111&public_key=generated-error-tracking-key" \ + "http://localhost:3001/api/v4/internal/error_tracking_allowed" +``` + +Example response: + +```json +{ "enabled": true } +``` + +### Known consumers + +- OpsTrace + ## Incrementing counter on pre-receive This is called from the Gitaly hooks increasing the reference counter @@ -559,6 +590,39 @@ curl --request POST --header "Gitlab-Kas-Api-Request: <JWT token>" \ --data '{ "uuids": ["102e8a0a-fe29-59bd-b46c-57c3e9bc6411", "5eb12985-0ed5-51f4-b545-fd8871dc2870"] }' ``` +### Scan Execution Policies + +Called from GitLab agent server (`kas`) to retrieve `scan_execution_policies` +configured for the project belonging to the agent token. GitLab `kas` uses +this to configure the agent to scan images in the Kubernetes cluster based on the policy. + +```plaintext +GET /internal/kubernetes/modules/starboard_vulnerability/scan_execution_policies +``` + +Example Request: + +```shell +curl --request GET --header "Gitlab-Kas-Api-Request: <JWT token>" \ + --header "Authorization: Bearer <agent token>" "http://localhost:3000/api/v4/internal/kubernetes/modules/starboard_vulnerability/scan_execution_policies" +``` + +Example response: + +```json +{ + "policies": [ + { + "name": "Policy", + "description": "Policy description", + "enabled": true, + "yaml": "---\nname: Policy\ndescription: 'Policy description'\nenabled: true\nactions:\n- scan: container_scanning\nrules:\n- type: pipeline\n branches:\n - main\n", + "updated_at": "2022-06-02T05:36:26+00:00" + } + ] +} +``` + ## Subscriptions The subscriptions endpoint is used by [CustomersDot](https://gitlab.com/gitlab-org/customers-gitlab-com) (`customers.gitlab.com`) @@ -763,7 +827,7 @@ Example response: ### Moving additional packs -Use a PATCH to move additional packs from one namespace to another. +Use a `PATCH` to move additional packs from one namespace to another. ```plaintext PATCH /namespaces/:id/minutes/move/:target_id @@ -816,7 +880,7 @@ Each array element contains: | Attribute | Type | Required | Description | |:-------------------|:-----------|:---------|:------------| | `namespace_id` | integer | yes | ID of the namespace to be reconciled | -| `next_reconciliation_date` | date | yes | Date when next reconciliation will happen | +| `next_reconciliation_date` | date | yes | Date of the next reconciliation | | `display_alert_from` | date | yes | Start date to display alert of upcoming reconciliation | Example request: diff --git a/doc/development/iterating_tables_in_batches.md b/doc/development/iterating_tables_in_batches.md index 8813fe560db..b4459b53efa 100644 --- a/doc/development/iterating_tables_in_batches.md +++ b/doc/development/iterating_tables_in_batches.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -31,7 +31,7 @@ User.each_batch(of: 10) do |relation| end ``` -This will end up producing queries such as: +This produces queries such as: ```plaintext User Load (0.7ms) SELECT "users"."id" FROM "users" WHERE ("users"."id" >= 41654) ORDER BY "users"."id" ASC LIMIT 1 OFFSET 1000 @@ -46,7 +46,7 @@ all of the arguments that `in_batches` supports. You should always use One should proceed with extra caution, and possibly avoid iterating over a column that can contain duplicate values. When you iterate over an attribute that is not unique, even with the applied max -batch size, there is no guarantee that the resulting batches will not surpass it. The following +batch size, there is no guarantee that the resulting batches do not surpass it. The following snippet demonstrates this situation when one attempt to select `Ci::Build` entries for users with `id` between `1` and `10,000`, the database returns `1 215 178` matching rows. @@ -67,7 +67,7 @@ SELECT "ci_builds".* FROM "ci_builds" WHERE "ci_builds"."type" = 'Ci::Build' AND even though the range size is limited to a certain threshold (`10,000` in the previous example) this threshold does not translate to the size of the returned dataset. That happens because when taking `n` possible values of attributes, one can't tell for sure that the number of records that contains -them will be less than `n`. +them is less than `n`. ## Column definition @@ -99,8 +99,8 @@ determines the data ranges (slices) and schedules the background jobs uses `each ## Efficient usage of `each_batch` -`EachBatch` helps to iterate over large tables. It's important to highlight that `EachBatch` is -not going to magically solve all iteration related performance problems and it might not help at +`EachBatch` helps to iterate over large tables. It's important to highlight that `EachBatch` +does not magically solve all iteration-related performance problems, and it might not help at all in some scenarios. From the database point of view, correctly configured database indexes are also necessary to make `EachBatch` perform well. @@ -108,7 +108,7 @@ also necessary to make `EachBatch` perform well. Let's consider that we want to iterate over the `users` table and print the `User` records to the standard output. The `users` table contains millions of records, thus running one query to fetch -the users will likely time out. +the users likely times out.  @@ -171,7 +171,7 @@ SELECT "users".* FROM "users" WHERE "users"."id" >= 1 AND "users"."id" < 302  Notice the `<` sign. Previously six items were read from the index and in this query, the last -value is "excluded". The query will look at the index to get the location of the five `user` +value is "excluded". The query looks at the index to get the location of the five `user` rows on the disk and read the rows from the table. The returned array is processed in Ruby. The first iteration is done. For the next iteration, the last `id` value is reused from the @@ -204,13 +204,13 @@ users.each_batch(of: 5) do |relation| end ``` -`each_batch` will produce the following SQL query for the start `id` value: +`each_batch` produces the following SQL query for the start `id` value: ```sql SELECT "users"."id" FROM "users" WHERE "users"."sign_in_count" = 0 ORDER BY "users"."id" ASC LIMIT 1 ``` -Selecting only the `id` column and ordering by `id` is going to "force" the database to use the +Selecting only the `id` column and ordering by `id` forces the database to use the index on the `id` (primary key index) column however, we also have an extra condition on the `sign_in_count` column. The column is not part of the index, so the database needs to look into the actual table to find the first matching row. @@ -225,7 +225,7 @@ The number of scanned rows depends on the data distribution in the table. In this particular example, the database had to read 10 rows (regardless of our batch size setting) to determine the first `id` value. In a "real-world" application it's hard to predict whether the -filtering is going to cause problems or not. In the case of GitLab, verifying the data on a +filtering causes problems or not. In the case of GitLab, verifying the data on a production replica is a good start, but keep in mind that data distribution on GitLab.com can be different from self-managed instances. @@ -289,7 +289,7 @@ CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count, id)  -The following index definition is not going to work well with `each_batch` (avoid). +The following index definition does not work well with `each_batch` (avoid). ```sql CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count) diff --git a/doc/development/licensed_feature_availability.md b/doc/development/licensed_feature_availability.md index 6df5c2164e8..09c32fc4244 100644 --- a/doc/development/licensed_feature_availability.md +++ b/doc/development/licensed_feature_availability.md @@ -22,9 +22,17 @@ it should be restricted on namespace scope. 1. Check using: ```ruby -project.feature_available?(:feature_symbol) +project.licensed_feature_available?(:feature_symbol) ``` +or + +```ruby +group.licensed_feature_available?(:feature_symbol) +``` + +For projects, `licensed_feature_available` delegates to its associated `namespace`. + ## Restricting global features (instance) However, for features such as [Geo](../administration/geo/index.md) and diff --git a/doc/development/logging.md b/doc/development/logging.md index 6a0b50d6970..749f85c9e2d 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -344,7 +344,7 @@ provides helper methods to track exceptions: 1. `Gitlab::ErrorTracking.track_exception`: this method only logs and sends exception to Sentry (if configured), 1. `Gitlab::ErrorTracking.log_exception`: this method only logs the exception, - and DOES NOT send the exception to Sentry, + and does not send the exception to Sentry, 1. `Gitlab::ErrorTracking.track_and_raise_for_dev_exception`: this method logs, sends exception to Sentry (if configured) and re-raises the exception for development and test environments. diff --git a/doc/development/maintenance_mode.md b/doc/development/maintenance_mode.md index a118d9cf0ad..f70cca1040e 100644 --- a/doc/development/maintenance_mode.md +++ b/doc/development/maintenance_mode.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/merge_request_concepts/index.md b/doc/development/merge_request_concepts/index.md index 90e8ff41368..8df0da5123e 100644 --- a/doc/development/merge_request_concepts/index.md +++ b/doc/development/merge_request_concepts/index.md @@ -1,7 +1,7 @@ --- type: reference, dev -stage: create -group: code_review +stage: Create +group: Code Review info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" --- diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index aebecd90574..c9b59ba66b5 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -52,9 +52,9 @@ work it needs to perform and how long it takes to complete: - Clean-ups, like removing unused columns. - Adding non-critical indices on high-traffic tables. - Adding non-critical indices that take a long time to create. -1. [**Background migrations.**](database/background_migrations.md) These aren't regular Rails migrations, but application code that is +1. [**Batched background migrations.**](database/batched_background_migrations.md) These aren't regular Rails migrations, but application code that is executed via Sidekiq jobs, although a post-deployment migration is used to schedule them. Use them only for data migrations that - exceed the timing guidelines for post-deploy migrations. Background migrations should _not_ change the schema. + exceed the timing guidelines for post-deploy migrations. Batched background migrations should _not_ change the schema. Use the following diagram to guide your decision, but keep in mind that it is just a tool, and the final outcome will always be dependent on the specific changes being made: @@ -495,7 +495,7 @@ def up end ``` -The RuboCop rule generally allows standard Rails migration methods, listed below. This example causes a Rubocop offense: +The RuboCop rule generally allows standard Rails migration methods, listed below. This example causes a RuboCop offense: ```ruby disable_ddl_transaction! @@ -898,6 +898,44 @@ def down end ``` +## Dropping a sequence + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88387) in GitLab 15.1. + +Dropping a sequence is uncommon, but you can use the `drop_sequence` method provided by the database team. + +Under the hood, it works like this: + +Remove a sequence: + +- Remove the default value if the sequence is actually used. +- Execute `DROP SEQUENCE`. + +Re-add a sequence: + +- Create the sequence, with the possibility of specifying the current value. +- Change the default value of the column. + +A Rails migration example: + +```ruby +class DropSequenceTest < Gitlab::Database::Migration[2.0] + def up + drop_sequence(:ci_pipelines_config, :pipeline_id, :ci_pipelines_config_pipeline_id_seq) + end + + def down + default_value = Ci::Pipeline.maximum(:id) + 10_000 + + add_sequence(:ci_pipelines_config, :pipeline_id, :ci_pipelines_config_pipeline_id_seq, default_value) + end +end +``` + +NOTE: +`add_sequence` should be avoided for columns with foreign keys. +Adding sequence to these columns is **only allowed** in the down method (restore previous schema state). + ## Integer column type By default, an integer column can hold up to a 4-byte (32-bit) number. That is diff --git a/doc/development/new_fe_guide/modules/widget_extensions.md b/doc/development/new_fe_guide/modules/widget_extensions.md index d3be8981abb..4bae0ac70c4 100644 --- a/doc/development/new_fe_guide/modules/widget_extensions.md +++ b/doc/development/new_fe_guide/modules/widget_extensions.md @@ -46,6 +46,7 @@ export default { methods: { fetchCollapsedData(props) {}, // Required: Fetches data required for collapsed state fetchFullData(props) {}, // Required: Fetches data for the full expanded content + fetchMultiData() {}, // Optional: Works in conjunction with `enablePolling` and allows polling multiple endpoints }, }; ``` @@ -232,6 +233,30 @@ export default { }; ``` +If the extension needs to poll multiple endpoints at the same time, then `fetchMultiData` +can be used to return an array of functions. A new `poll` object is created for each +endpoint and they are polled separately. After all endpoints are resolved, polling is +stopped and `setCollapsedData` is called with an array of `response.data`. + +```javascript +export default { + //... + enablePolling: true + methods: { + fetchMultiData() { + return [ + () => axios.get(this.reportPath1), + () => axios.get(this.reportPath2), + () => axios.get(this.reportPath3) + }, + }, +}; +``` + +**Important** The function needs to return a `Promise` that resolves the `response` object. +The implementation relies on the `POLL-INTERVAL` header to keep polling, therefore it is +important not to alter the status code and headers. + ### Errors If `fetchCollapsedData()` or `fetchFullData()` methods throw an error: diff --git a/doc/development/omnibus.md b/doc/development/omnibus.md index dc83b0ea257..b62574e34e5 100644 --- a/doc/development/omnibus.md +++ b/doc/development/omnibus.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md index 00ce15fcc10..7cd3d4fb208 100644 --- a/doc/development/ordering_table_columns.md +++ b/doc/development/ordering_table_columns.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -24,15 +24,15 @@ The first column is a 4-byte integer. The next is text of variable length. The bytes. To meet the alignment requirements, four zeros are to be added right after the first column, so `id` occupies 4 bytes, then 4 bytes of alignment padding, and only next `name` is being stored. Therefore, in this case, 8 bytes -will be spent for storing a 4-byte integer. +are spent for storing a 4-byte integer. The space between rows is also subject to alignment padding. The `user_id` -column takes only 4 bytes, and on 64-bit platform, 4 zeroes will be added for +column takes only 4 bytes, and on 64-bit platform, 4 zeroes are added for alignment padding, to allow storing the next row beginning with the "clear" word. As a result, the actual size of each column would be (omitting variable length data and 24-byte tuple header): 8 bytes, variable, 8 bytes. This means that -each row will require at least 16 bytes for the two 4-byte integers. If a table +each row requires at least 16 bytes for the two 4-byte integers. If a table has a few rows this is not an issue. However, once you start storing millions of rows you can save space by using a different order. For the above example, the ideal column order would be the following: @@ -49,7 +49,7 @@ or In these examples, the `id` and `user_id` columns are packed together, which means we only need 8 bytes to store _both_ of them. This in turn means each row -will require 8 bytes less space. +requires 8 bytes less space. Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. We are using `integer` in the examples to showcase a more realistic reordering scenario. @@ -57,7 +57,7 @@ We are using `integer` in the examples to showcase a more realistic reordering s ## Type Sizes While the [PostgreSQL documentation](https://www.postgresql.org/docs/current/datatype.html) contains plenty -of information we will list the sizes of common types here so it's easier to +of information we list the sizes of common types here so it's easier to look them up. Here "word" refers to the word size, which is 4 bytes for a 32 bits platform and 8 bytes for a 64 bits platform. @@ -69,7 +69,7 @@ bits platform and 8 bytes for a 64 bits platform. | `real` | 4 bytes | 1 word | | `double precision` | 8 bytes | 8 bytes | | `boolean` | 1 byte | not needed | -| `text` / `string` | variable, 1 byte plus the data | 1 word | +| `text` / `string` | variable, 1 byte plus the data | 1 word | | `bytea` | variable, 1 or 4 bytes plus the data | 1 word | | `timestamp` | 8 bytes | 8 bytes | | `timestamptz` | 8 bytes | 8 bytes | @@ -77,7 +77,7 @@ bits platform and 8 bytes for a 64 bits platform. A "variable" size means the actual size depends on the value being stored. If PostgreSQL determines this can be embedded directly into a row it may do so, but -for very large values it will store the data externally and store a pointer (of +for very large values it stores the data externally and store a pointer (of 1 word in size) in the column. Because of this variable sized columns should always be at the end of a table. diff --git a/doc/development/packages.md b/doc/development/packages.md index 6526bdd45a1..a79f5f09677 100644 --- a/doc/development/packages.md +++ b/doc/development/packages.md @@ -1,302 +1,11 @@ --- -stage: Package -group: Package -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'packages/index.md' +remove_date: '2022-08-19' --- -# Packages +This document was moved to [another location](packages/index.md). -This document guides you through adding support to GitLab for a new a [package management system](../administration/packages/index.md). - -See the already supported formats in the [Packages & Registries documentation](../user/packages/index.md) - -It is possible to add a new format with only backend changes. -This guide is superficial and does not cover the way the code should be written. -However, you can find a good example by looking at the following merge requests: - -- [npm registry support](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8673) -- [Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6607) -- [Instance-level API for Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8757) -- [NuGet group-level API](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) - -## General information - -The existing database model requires the following: - -- Every package belongs to a project. -- Every package file belongs to a package. -- A package can have one or more package files. -- The package model is based on storing information about the package and its version. - -### API endpoints - -Package systems work with GitLab via API. For example `lib/api/npm_project_packages.rb` -implements API endpoints to work with npm clients. So, the first thing to do is to -add a new `lib/api/your_name_project_packages.rb` file with API endpoints that are -necessary to make the package system client to work. Usually that means having -endpoints like: - -- GET package information. -- GET package file content. -- PUT upload package. - -Since the packages belong to a project, it's expected to have project-level endpoint (remote) -for uploading and downloading them. For example: - -```plaintext -GET https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/ -PUT https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/ -``` - -Group-level and instance-level endpoints should only be considered after the project-level endpoint is available in production. - -#### Remote hierarchy - -Packages are scoped within various levels of access, which is generally configured by setting your remote. A -remote endpoint may be set at the project level, meaning when installing packages, only packages belonging to that -project are visible. Alternatively, a group-level endpoint may be used to allow visibility to all packages -within a given group. Lastly, an instance-level endpoint can be used to allow visibility to all packages within an -entire GitLab instance. - -As an MVC, we recommend beginning with a project-level endpoint. A typical iteration plan for remote hierarchies is to go from: - -- Publish and install in a project -- Install from a group -- Publish and install in an Instance (this is for Self-Managed customers) - -Using instance-level endpoints requires [stricter naming conventions](#naming-conventions). - -NOTE: -Composer package naming scope is Instance Level. - -### Naming conventions - -To avoid name conflict for instance-level endpoints you must define a package naming convention -that gives a way to identify the project that the package belongs to. This generally involves using the project -ID or full project path in the package name. See -[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes) as an example. - -For group and project-level endpoints, naming can be less constrained and it is up to the group and project -members to be certain that there is no conflict between two package names. However, the system should prevent -a user from reusing an existing name within a given scope. - -Otherwise, naming should follow the package manager's naming conventions and include a validation in the `package.md` -model for that package type. - -### Services and finders - -Logic for performing tasks such as creating package or package file records or finding packages should not live -within the API file, but should live in services and finders. Existing services and finders should be used or -extended when possible to keep the common package logic grouped as much as possible. - -### Configuration - -GitLab has a `packages` section in its configuration file (`gitlab.rb`). -It applies to all package systems supported by GitLab. Usually you don't need -to add anything there. - -Packages can be configured to use object storage, therefore your code must support it. - -## MVC Approach - -The way new package systems are integrated in GitLab is using an [MVC](https://about.gitlab.com/handbook/values/#minimum-viable-change-mvc). Therefore, the first iteration should support the bare minimum user actions: - -- Authentication with a GitLab job, personal access, project access, or deploy token -- Uploading a package and displaying basic metadata in the user interface -- Pulling a package -- Required actions - -Required actions are all the additional requests that GitLab needs to handle so the corresponding package manager CLI can work properly. It could be a search feature or an endpoint providing meta information about a package. For example: - -- For NuGet, the search request was implemented during the first MVC iteration, to support Visual Studio. -- For npm, there is a metadata endpoint used by `npm` to get the tarball URL. - -For the first MVC iteration, it's recommended to stay at the project level of the [remote hierarchy](#remote-hierarchy). Other levels can be tackled with [future Merge Requests](#future-work). - -There are usually 2 phases for the MVC: - -- [Analysis](#analysis) -- [Implementation](#implementation) - -### Keep iterations small - -When implementing a new package manager, it is tempting to create one large merge request containing all of the -necessary endpoints and services necessary to support basic usage. Instead, put the -API endpoints behind a [feature flag](feature_flags/index.md) and -submit each endpoint or behavior (download, upload, etc) in a different merge request to shorten the review -process. - -### Analysis - -During this phase, the idea is to collect as much information as possible about the API used by the package system. Here some aspects that can be useful to include: - -- **Authentication**: What authentication mechanisms are available (OAuth, Basic - Authorization, other). Keep in mind that GitLab users often want to use their - [Personal Access Tokens](../user/profile/personal_access_tokens.md). - Although not needed for the MVC first iteration, the [CI/CD job tokens](../ci/jobs/ci_job_token.md) - have to be supported at some point in the future. -- **Requests**: Which requests are needed to have a working MVC. Ideally, produce - a list of all the requests needed for the MVC (including required actions). Further - investigation could provide an example for each request with the request and the response bodies. -- **Upload**: Carefully analyze how the upload process works. This is likely the most - complex request to implement. A detailed analysis is desired here as uploads can be - encoded in different ways (body or multipart) and can even be in a totally different - format (for example, a JSON structure where the package file is a Base64 value of - a particular field). These different encodings lead to slightly different implementations - on GitLab and GitLab Workhorse. For more detailed information, review [file uploads](#file-uploads). -- **Endpoints**: Suggest a list of endpoint URLs to implement in GitLab. -- **Split work**: Suggest a list of changes to do to incrementally build the MVC. - This gives a good idea of how much work there is to be done. Here is an example - list that would need to be adapted on a case by case basis: - 1. Empty file structure (API file, base service for this package) - 1. Authentication system for "logging in" to the package manager - 1. Identify metadata and create applicable tables - 1. Workhorse route for [object storage direct upload](uploads/index.md#direct-upload) - 1. Endpoints required for upload/publish - 1. Endpoints required for install/download - 1. Endpoints required for required actions - -The analysis usually takes a full milestone to complete, though it's not impossible to start the implementation in the same milestone. - -In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Workhorse is already ready when the upload request is implemented on the rails backend. - -### Implementation - -The implementation of the different Merge Requests varies between different package system integrations. Contributors should take into account some important aspects of the implementation phase. - -#### Authentication - -The MVC must support [Personal Access Tokens](../user/profile/personal_access_tokens.md) right from the start. We currently support two options for these tokens: OAuth and Basic Access. - -OAuth authentication is already supported. You can see an example in the [npm API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/npm_project_packages.rb). - -[Basic Access authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) -support is done by overriding a specific function in the API helpers, like -[this example in the Conan API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/conan_packages.rb). -For this authentication mechanism, keep in mind that some clients can send an unauthenticated -request first, wait for the 401 Unauthorized response with the [`WWW-Authenticate`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/WWW-Authenticate) -field, then send an updated (authenticated) request. This case is more involved as -GitLab needs to handle the 401 Unauthorized response. The [NuGet API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/nuget_packages.rb) -supports this case. - -#### Authorization - -There are project and group level permissions for `read_package`, `create_package`, and `destroy_package`. Each -endpoint should -[authorize the requesting user](https://gitlab.com/gitlab-org/gitlab/-/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/conan_packages.rb) -against the project or group before continuing. - -#### Database and handling metadata - -The current database model allows you to store a name and a version for each package. -Every time you upload a new package, you can either create a new record of `Package` -or add files to existing record. `PackageFile` should be able to store all file-related -information like the file `name`, `side`, `sha1`, and so on. - -If there is specific data necessary to be stored for only one package system support, -consider creating a separate metadata model. See `packages_maven_metadata` table -and `Packages::Maven::Metadatum` model as an example for package specific data, and `packages_conan_file_metadata` table -and `Packages::Conan::FileMetadatum` model as an example for package file specific data. - -If there is package specific behavior for a given package manager, add those methods to the metadata models and -delegate from the package model. - -Note that the existing package UI only displays information within the `packages_packages` and `packages_package_files` -tables. If the data stored in the metadata tables need to be displayed, a ~frontend change is required. - -#### File uploads - -File uploads should be handled by GitLab Workhorse using object accelerated uploads. What this means is that -the workhorse proxy that checks all incoming requests to GitLab intercept the upload request, -upload the file, and forward a request to the main GitLab codebase only containing the metadata -and file location rather than the file itself. An overview of this process can be found in the -[development documentation](uploads/index.md#direct-upload). - -In terms of code, this means a route must be added to the -[GitLab Workhorse project](https://gitlab.com/gitlab-org/gitlab-workhorse) for each upload endpoint being added -(instance, group, project). [This merge request](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/412/diffs) -demonstrates adding an instance-level endpoint for Conan to workhorse. You can also see the Maven project level endpoint -implemented in the same file. - -Once the route has been added, you must add an additional `/authorize` version of the upload endpoint to your API file. -[This example](https://gitlab.com/gitlab-org/gitlab/-/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/maven_packages.rb#L164) -shows the additional endpoint added for Maven. The `/authorize` endpoint verifies and authorizes the request from workhorse, -then the normal upload endpoint is implemented below, consuming the metadata that workhorse provides in order to -create the package record. Workhorse provides a variety of file metadata such as type, size, and different checksum formats. - -For testing purposes, you may want to [enable object storage](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/object_storage.md) -in your local development environment. - -#### File size limits - -Files uploaded to the GitLab Package Registry are [limited by format](../administration/instance_limits.md#package-registry-limits). -On GitLab.com, these are typically set to 5GB to help prevent timeout issues and abuse. - -When a new package type is added to the `Packages::Package` model, a size limit must be added -similar to [this example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52639/diffs#382f879fb09b0212e3cedd99e6c46e2083867216), -or the [related test](https://gitlab.com/gitlab-org/gitlab/-/blob/fe4ba43766781371cebfacd78364a1de762917cd/spec/models/packages/package_spec.rb#L761) -must be updated if file size limits do not apply. The only reason a size limit does not apply is if -the package format does not upload and store package files. - -#### Rate Limits on GitLab.com - -Package manager clients can make rapid requests that exceed the -[GitLab.com standard API rate limits](../user/gitlab_com/index.md#gitlabcom-specific-rate-limits). -This results in a `429 Too Many Requests` error. - -We have opened a set of paths to allow higher rate limits. Unless it is not possible, -new package managers should follow these conventions so they can take advantage of the -expanded package rate limit. - -These route prefixes guarantee a higher rate limit: - -```plaintext -/api/v4/packages/ -/api/v4/projects/:project_id/packages/ -/api/v4/groups/:group_id/-/packages/ -``` - -### MVC Checklist - -When adding support to GitLab for a new package manager, the first iteration must contain the -following features. You can add the features through many merge requests as needed, but all the -features must be implemented when the feature flag is removed. - -- Project-level API -- Push event tracking -- Pull event tracking -- Authentication with Personal Access Tokens -- Authentication with Job Tokens -- Authentication with Deploy Tokens (group and project) -- File size [limit](#file-size-limits) -- File format guards (only accept valid file formats for the package type) -- Name regex with validation -- Version regex with validation -- Workhorse route for [accelerated](uploads/working_with_uploads.md) uploads -- Background workers for extracting package metadata (if applicable) -- Documentation (how to use the feature) -- API Documentation (individual endpoints with curl examples) -- Seeding in [`db/fixtures/development/26_packages.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/fixtures/development/26_packages.rb) -- Update the [runbook](https://gitlab.com/gitlab-com/runbooks/-/blob/31fb4959e89db25fddf865bc81734c222daf32dd/dashboards/stage-groups/package.dashboard.jsonnet#L74) for the Grafana charts -- End-to-end feature tests for (at the minimum) publishing and installing a package - -### Future Work - -While working on the MVC, contributors might find features that are not mandatory for the MVC but can provide a better user experience. It's generally a good idea to keep an eye on those and open issues. - -Here are some examples - -1. Endpoints required for search -1. Front end updates to display additional package information and metadata -1. Limits on file sizes -1. Tracking for metrics -1. Read more metadata fields from the package to make it available to the front end. For example, it's usual to be able to tag a package. Those tags can be read and saved by backend and then displayed on the packages UI. -1. Endpoints for the upper levels of the [remote hierarchy](#remote-hierarchy). This step might need to create a [naming convention](#naming-conventions) - -## Exceptions - -This documentation is just guidelines on how to implement a package manager to match the existing structure and logic -already present within GitLab. While the structure is intended to be extendable and flexible enough to allow for -any given package manager, if there is good reason to stray due to the constraints or needs of a given package -manager, then it should be raised and discussed within the implementation issue or merge request to work towards -the most efficient outcome. +<!-- This redirect file can be deleted after <2022-08-19>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/packages/index.md b/doc/development/packages/index.md new file mode 100644 index 00000000000..55deaa229ba --- /dev/null +++ b/doc/development/packages/index.md @@ -0,0 +1,25 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Package Registry Development + +Development and Architectural documentation for the package registry + +- [Developing a new format](new_format_development.md) +- [Settings](settings.md) +- [Structure / Schema](structure.md) +- API documentation + - [Composer](../../api/packages/composer.md) + - [Conan](../../api/packages/conan.md) + - [Debian](../../api/packages/debian.md) + - [Generic](../../user/packages/generic_packages/index.md) + - [Go Proxy](../../api/packages/go_proxy.md) + - [Helm](../../api/packages/helm.md) + - [Maven](../../api/packages/maven.md) + - [npm](../../api/packages/npm.md) + - [NuGet](../../api/packages/nuget.md) + - [PyPI](../../api/packages/pypi.md) + - [Ruby Gems](../../api/packages/rubygems.md) diff --git a/doc/development/packages/new_format_development.md b/doc/development/packages/new_format_development.md new file mode 100644 index 00000000000..f7d02f9160b --- /dev/null +++ b/doc/development/packages/new_format_development.md @@ -0,0 +1,302 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Developing support for a new package format + +This document guides you through adding support to GitLab for a new a [package management system](../../administration/packages/index.md). + +See the already supported formats in the [Packages & Registries documentation](../../user/packages/index.md) + +It is possible to add a new format with only backend changes. +This guide is superficial and does not cover the way the code should be written. +However, you can find a good example by looking at the following merge requests: + +- [npm registry support](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8673) +- [Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6607) +- [Instance-level API for Maven repository](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/8757) +- [NuGet group-level API](https://gitlab.com/gitlab-org/gitlab/-/issues/36423) + +## General information + +The existing database model requires the following: + +- Every package belongs to a project. +- Every package file belongs to a package. +- A package can have one or more package files. +- The package model is based on storing information about the package and its version. + +### API endpoints + +Package systems work with GitLab via API. For example `lib/api/npm_project_packages.rb` +implements API endpoints to work with npm clients. So, the first thing to do is to +add a new `lib/api/your_name_project_packages.rb` file with API endpoints that are +necessary to make the package system client work. Usually that means having +endpoints like: + +- GET package information. +- GET package file content. +- PUT upload package. + +Because the packages belong to a project, it's expected to have project-level endpoint (remote) +for uploading and downloading them. For example: + +```plaintext +GET https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/ +PUT https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/ +``` + +Group-level and instance-level endpoints should only be considered after the project-level endpoint is available in production. + +#### Remote hierarchy + +Packages are scoped within various levels of access, which is generally configured by setting your remote. A +remote endpoint may be set at the project level, meaning when installing packages, only packages belonging to that +project are visible. Alternatively, a group-level endpoint may be used to allow visibility to all packages +in a given group. Lastly, an instance-level endpoint can be used to allow visibility to all packages in an +entire GitLab instance. + +As an MVC, we recommend beginning with a project-level endpoint. A typical iteration plan for remote hierarchies is to go from: + +- Publish and install in a project +- Install from a group +- Publish and install in an Instance (this is for Self-Managed customers) + +Using instance-level endpoints requires [stricter naming conventions](#naming-conventions). + +NOTE: +Composer package naming scope is Instance Level. + +### Naming conventions + +To avoid name conflict for instance-level endpoints you must define a package naming convention +that gives a way to identify the project that the package belongs to. This generally involves using the project +ID or full project path in the package name. See +[Conan's naming convention](../../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes) as an example. + +For group and project-level endpoints, naming can be less constrained and it is up to the group and project +members to be certain that there is no conflict between two package names. However, the system should prevent +a user from reusing an existing name within a given scope. + +Otherwise, naming should follow the package manager's naming conventions and include a validation in the `package.md` +model for that package type. + +### Services and finders + +Logic for performing tasks such as creating package or package file records or finding packages should not live +in the API file, but should live in services and finders. Existing services and finders should be used or +extended when possible to keep the common package logic grouped as much as possible. + +### Configuration + +GitLab has a `packages` section in its configuration file (`gitlab.rb`). +It applies to all package systems supported by GitLab. Usually you don't need +to add anything there. + +Packages can be configured to use object storage, therefore your code must support it. + +## MVC Approach + +The way new package systems are integrated in GitLab is using an [MVC](https://about.gitlab.com/handbook/values/#minimum-viable-change-mvc). Therefore, the first iteration should support the bare minimum user actions: + +- Authentication with a GitLab job, personal access, project access, or deploy token +- Uploading a package and displaying basic metadata in the user interface +- Pulling a package +- Required actions + +Required actions are all the additional requests that GitLab must handle so the corresponding package manager CLI can work properly. It could be a search feature or an endpoint providing meta information about a package. For example: + +- For NuGet, the search request was implemented during the first MVC iteration, to support Visual Studio. +- For npm, there is a metadata endpoint used by `npm` to get the tarball URL. + +For the first MVC iteration, it's recommended to stay at the project level of the [remote hierarchy](#remote-hierarchy). Other levels can be tackled with [future Merge Requests](#future-work). + +The MVC usually has two phases: + +- [Analysis](#analysis) +- [Implementation](#implementation) + +### Keep iterations small + +When implementing a new package manager, it is tempting to create one large merge request containing all of the +necessary endpoints and services necessary to support basic usage. Instead: + +1. Put the API endpoints behind a [feature flag](../feature_flags/index.md). +1. Submit each endpoint or behavior (download, upload, etc) in a different merge request to shorten the review process. + +### Analysis + +During this phase, the idea is to collect as much information as possible about the API used by the package system. Here some aspects that can be useful to include: + +- **Authentication**: What authentication mechanisms are available (OAuth, Basic + Authorization, other). Keep in mind that GitLab users often want to use their + [Personal Access Tokens](../../user/profile/personal_access_tokens.md). + Although not needed for the MVC first iteration, the [CI/CD job tokens](../../ci/jobs/ci_job_token.md) + have to be supported at some point in the future. +- **Requests**: Which requests are needed to have a working MVC. Ideally, produce + a list of all the requests needed for the MVC (including required actions). Further + investigation could provide an example for each request with the request and the response bodies. +- **Upload**: Carefully analyze how the upload process works. This request is likely the most + complex to implement. A detailed analysis is desired here as uploads can be + encoded in different ways (body or multipart) and can even be in a totally different + format (for example, a JSON structure where the package file is a Base64 value of + a particular field). These different encodings lead to slightly different implementations + on GitLab and GitLab Workhorse. For more detailed information, review [file uploads](#file-uploads). +- **Endpoints**: Suggest a list of endpoint URLs to implement in GitLab. +- **Split work**: Suggest a list of changes to do to incrementally build the MVC. + This gives a good idea of how much work there is to be done. Here is an example + list that must be adapted on a case by case basis: + 1. Empty file structure (API file, base service for this package) + 1. Authentication system for "logging in" to the package manager + 1. Identify metadata and create applicable tables + 1. Workhorse route for [object storage direct upload](../uploads/index.md#direct-upload) + 1. Endpoints required for upload/publish + 1. Endpoints required for install/download + 1. Endpoints required for required actions + +The analysis usually takes a full milestone to complete, though it's not impossible to start the implementation in the same milestone. + +In particular, the upload request can have some [requirements in the GitLab Workhorse project](#file-uploads). This project has a different release cycle than the rails backend. It's **strongly** recommended that you open an issue there as soon as the upload request analysis is done. This way GitLab Workhorse is already ready when the upload request is implemented on the rails backend. + +### Implementation + +The implementation of the different Merge Requests varies between different package system integrations. Contributors should take into account some important aspects of the implementation phase. + +#### Authentication + +The MVC must support [Personal Access Tokens](../../user/profile/personal_access_tokens.md) right from the start. We support two options for these tokens: OAuth and Basic Access. + +OAuth authentication is already supported. You can see an example in the [npm API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/npm_project_packages.rb). + +[Basic Access authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication) +support is done by overriding a specific function in the API helpers, like +[this example in the Conan API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/conan_packages.rb). +For this authentication mechanism, keep in mind that some clients can send an unauthenticated +request first, wait for the 401 Unauthorized response with the [`WWW-Authenticate`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/WWW-Authenticate) +field, then send an updated (authenticated) request. This case is more involved as +GitLab must handle the `401 Unauthorized` response. The [NuGet API](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/nuget_packages.rb) +supports this case. + +#### Authorization + +Project and group level permissions exist for `read_package`, `create_package`, and `destroy_package`. Each +endpoint should +[authorize the requesting user](https://gitlab.com/gitlab-org/gitlab/-/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/conan_packages.rb) +against the project or group before continuing. + +#### Database and handling metadata + +The current database model allows you to store a name and a version for each package. +Every time you upload a new package, you can either create a new record of `Package` +or add files to existing record. `PackageFile` should be able to store all file-related +information like the file `name`, `side`, `sha1`, and so on. + +If there is specific data necessary to be stored for only one package system support, +consider creating a separate metadata model. See `packages_maven_metadata` table +and `Packages::Maven::Metadatum` model as an example for package specific data, and `packages_conan_file_metadata` table +and `Packages::Conan::FileMetadatum` model as an example for package file specific data. + +If there is package specific behavior for a given package manager, add those methods to the metadata models and +delegate from the package model. + +The existing package UI only displays information in the `packages_packages` and `packages_package_files` +tables. If the data stored in the metadata tables must be displayed, a `~frontend` change is required. + +#### File uploads + +File uploads should be handled by GitLab Workhorse using object accelerated uploads. What this means is that +the workhorse proxy that checks all incoming requests to GitLab intercept the upload request, +upload the file, and forward a request to the main GitLab codebase only containing the metadata +and file location rather than the file itself. An overview of this process can be found in the +[development documentation](../uploads/index.md#direct-upload). + +In terms of code, this means a route must be added to the +[GitLab Workhorse project](https://gitlab.com/gitlab-org/gitlab-workhorse) for each upload endpoint being added +(instance, group, project). [This merge request](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/412/diffs) +demonstrates adding an instance-level endpoint for Conan to workhorse. You can also see the Maven project level endpoint +implemented in the same file. + +After the route has been added, you must add an additional `/authorize` version of the upload endpoint to your API file. +[This example](https://gitlab.com/gitlab-org/gitlab/-/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/maven_packages.rb#L164) +shows the additional endpoint added for Maven. The `/authorize` endpoint verifies and authorizes the request from workhorse, +then the normal upload endpoint is implemented below, consuming the metadata that Workhorse provides to +create the package record. Workhorse provides a variety of file metadata such as type, size, and different checksum formats. + +For testing purposes, you may want to [enable object storage](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/object_storage.md) +in your local development environment. + +#### File size limits + +Files uploaded to the GitLab Package Registry are [limited by format](../../administration/instance_limits.md#package-registry-limits). +On GitLab.com, these are typically set to 5GB to help prevent timeout issues and abuse. + +When a new package type is added to the `Packages::Package` model, a size limit must be added +similar to [this example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52639/diffs#382f879fb09b0212e3cedd99e6c46e2083867216), +or the [related test](https://gitlab.com/gitlab-org/gitlab/-/blob/fe4ba43766781371cebfacd78364a1de762917cd/spec/models/packages/package_spec.rb#L761) +must be updated if file size limits do not apply. The only reason a size limit does not apply is if +the package format does not upload and store package files. + +#### Rate Limits on GitLab.com + +Package manager clients can make rapid requests that exceed the +[GitLab.com standard API rate limits](../../user/gitlab_com/index.md#gitlabcom-specific-rate-limits). +This results in a `429 Too Many Requests` error. + +We have opened a set of paths to allow higher rate limits. Unless it is not possible, +new package managers should follow these conventions so they can take advantage of the +expanded package rate limit. + +These route prefixes guarantee a higher rate limit: + +```plaintext +/api/v4/packages/ +/api/v4/projects/:project_id/packages/ +/api/v4/groups/:group_id/-/packages/ +``` + +### MVC Checklist + +When adding support to GitLab for a new package manager, the first iteration must contain the +following features. You can add the features through many merge requests as needed, but all the +features must be implemented when the feature flag is removed. + +- Project-level API +- Push event tracking +- Pull event tracking +- Authentication with Personal Access Tokens +- Authentication with Job Tokens +- Authentication with Deploy Tokens (group and project) +- File size [limit](#file-size-limits) +- File format guards (only accept valid file formats for the package type) +- Name regex with validation +- Version regex with validation +- Workhorse route for [accelerated](../uploads/working_with_uploads.md) uploads +- Background workers for extracting package metadata (if applicable) +- Documentation (how to use the feature) +- API Documentation (individual endpoints with curl examples) +- Seeding in [`db/fixtures/development/26_packages.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/fixtures/development/26_packages.rb) +- Update the [runbook](https://gitlab.com/gitlab-com/runbooks/-/blob/31fb4959e89db25fddf865bc81734c222daf32dd/dashboards/stage-groups/package.dashboard.jsonnet#L74) for the Grafana charts +- End-to-end feature tests for (at the minimum) publishing and installing a package + +### Future Work + +While working on the MVC, contributors might find features that are not mandatory for the MVC but can provide a better user experience. It's generally a good idea to keep an eye on those and open issues. + +Here are some examples + +1. Endpoints required for search +1. Front end updates to display additional package information and metadata +1. Limits on file sizes +1. Tracking for metrics +1. Read more metadata fields from the package to make it available to the front end. For example, it's usual to be able to tag a package. Those tags can be read and saved by backend and then displayed on the packages UI. +1. Endpoints for the upper levels of the [remote hierarchy](#remote-hierarchy). This step might require you to create a [naming convention](#naming-conventions) + +## Exceptions + +This documentation is just guidelines on how to implement a package manager to match the existing structure and logic +already present in GitLab. While the structure is intended to be extendable and flexible enough to allow for +any given package manager, if there is good reason to stray due to the constraints or needs of a given package +manager, then it should be raised and discussed in the implementation issue or merge request to work towards +the most efficient outcome. diff --git a/doc/development/packages/settings.md b/doc/development/packages/settings.md new file mode 100644 index 00000000000..37961c0504c --- /dev/null +++ b/doc/development/packages/settings.md @@ -0,0 +1,82 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Package Settings + +This page includes an exhaustive list of settings related to and maintained by the package stage. + +## Instance Settings + +### Package Registry + +Setting | Table | Description +------- | ----- | ----------- +`npm_package_requests_forwarding` | `application_settings` | Enables or disables npm package forwarding at the instance level. +`pypi_package_requests_forwarding` | `application_settings` | Enables or disables PyPI package forwarding at the instance level. +`packages_cleanup_package_file_worker_capacity` | `application_settings` | Number of concurrent workers allowed for package file cleanup. +`throttle_unauthenticated_packages_api_requests_per_period` | `application_settings` | Request limit for unauthenticated package API requests in the period defined by `throttle_unauthenticated_packages_api_period_in_seconds`. +`throttle_unauthenticated_packages_api_period_in_seconds` | `application_settings` | Period in seconds to measure unauthenticated package API requests. +`throttle_authenticated_packages_api_requests_per_period` | `application_settings` | Request limit for authenticated package API requests in the period defined by `throttle_authenticated_packages_api_period_in_seconds`. +`throttle_authenticated_packages_api_period_in_seconds` | `application_settings` | Period in seconds to measure authenticated package API requests. +`throttle_unauthenticated_packages_api_enabled` | `application_settings` +`throttle_authenticated_packages_api_enabled` | `application_settings` | Enables or disables request limits/throttling for the package API. +`conan_max_file_size` | `plan_limits` | Maximum file size for a Conan package file. +`maven_max_file_size` | `plan_limits` | Maximum file size for a Maven package file. +`npm_max_file_size` | `plan_limits` | Maximum file size for an npm package file. +`nuget_max_file_size` | `plan_limits` | Maximum file size for a NuGet package file. +`pypi_max_file_size` | `plan_limits` | Maximum file size for a PyPI package file. +`generic_packages_max_file_size` | `plan_limits` | Maximum file size for a generic package file. +`golang_max_file_size` | `plan_limits` | Maximum file size for a GoProxy package file. +`debian_max_file_size` | `plan_limits` | Maximum file size for a Debian package file. +`rubygems_max_file_size` | `plan_limits` | Maximum file size for a RubyGems package file. +`terraform_module_max_file_size` | `plan_limits` | Maximum file size for a Terraform package file. +`helm_max_file_size` | `plan_limits` | Maximum file size for a Helm package file. + +### Container Registry + +Setting | Table | Description +------- | ----- | ----------- +`container_registry_token_expire_delay` | `application_settings` | The time in minutes before the container registry auth token (JWT) expires. +`container_expiration_policies_enable_historic_entries` | `application_settings` | Allow or prevent projects older than 12.8 to use container cleanup policies. +`container_registry_vendor` | `application_settings` | The vendor of the container registry. `gitlab` for the GitLab container registry, other values for external registries. +`container_registry_version` | `application_settings` | The current version of the container registry. +`container_registry_features` | `application_settings` | Features supported by the connected container registry. For example, tag deletion. +`container_registry_delete_tags_service_timeout` | `application_settings` | The maximum time (in seconds) that the cleanup process can take to delete a batch of tags. +`container_registry_expiration_policies_worker_capacity` | `application_settings` | Number of concurrent container image cleanup policy workers allowed. +`container_registry_cleanup_tags_service_max_list_size` | `application_settings` | The maximum number of tags that can be deleted in a cleanup policy single execution. Additional tags must be deleted in another execution. +`container_registry_expiration_policies_caching` | `application_settings` | Enable or disable tag creation timestamp caching during execution of cleanup policies. +`container_registry_import_max_tags_count` | `application_settings` | Defines what is a the maximum amount of tags that we accept to migrate. +`container_registry_import_max_retries` | `application_settings` | The maximum amount of retries done on a migration that is aborted. +`container_registry_import_start_max_retries` | `application_settings` | The maximum amount of requests to start an import step that is sent to the Container Registry API. +`container_registry_import_max_step_duration` | `application_settings` | The maximum amount of seconds before an ongoing migration is considered as stale. +`container_registry_import_target_plan` | `application_settings` | The target subscription plan on which we're intend to pick container repositories. +`container_registry_import_created_before` | `application_settings` | Only image repositories created before this timestamp are eligible for the migration. +`container_registry_pre_import_timeout` | `application_settings` | The timeout for long running `pre_imports` before they are canceled by the `GuardWorker`. +`container_registry_import_timeout` | `application_settings` | The timeout for long running imports before they are canceled by the `GuardWorker`. +`dependency_proxy_ttl_group_policy_worker_capacity` | `application_settings` | Number of concurrent dependency proxy cleanup policy workers allowed. + +## Namespace/Group Settings + +Setting | Table | Description +------- | ----- | ----------- +`maven_duplicates_allowed` | `namespace_package_settings` | Allow or prevent duplicate Maven packages. +`maven_duplicate_exception_regex` | `namespace_package_settings` | Regex defining Maven packages that are allowed to be duplicate when duplicates are not allowed. This matches the name and version of the package. +`generic_duplicates_allowed` | `namespace_package_settings` | Allow or prevent duplicate generic packages. +`generic_duplicate_exception_regex` | `namespace_package_settings` | Regex defining generic packages that are allowed to be duplicate when duplicates are not allowed. +Dependency Proxy Cleanup Policies - `ttl` | `dependency_proxy_image_ttl_group_policies` | Number of days to retain an unused Dependency Proxy file before it is removed. +Dependency Proxy - `enabled` | `dependency_proxy_image_ttl_group_policies` | Enable or disable the Dependency Proxy cleanup policy. + +## Project Settings + +Setting | Table | Description +------- | ----- | ----------- +Container Cleanup Policies - `next_run_at` | `container_expiration_policies` | When the project qualifies for the next container cleanup policy cron worker. +Container Cleanup Policies - `name_regex` | `container_expiration_policies` | Regex defining image names to remove with the container cleanup policy. +Container Cleanup Policies - `cadence` | `container_expiration_policies` | How often the container cleanup policy should run. +Container Cleanup Policies - `older_than` | `container_expiration_policies` | Age of images to remove with the container cleanup policy. +Container Cleanup Policies - `keep_n` | `container_expiration_policies` | Number of images to retain in a container cleanup policy. +Container Cleanup Policies - `enabled` | `container_expiration_policies` | Enable or disable a container cleanup policy. +Container Cleanup Policies - `name_regex_keep` | `container_expiration_policies` | Regex defining image names to always keep regardless of other rules with the container cleanup policy. diff --git a/doc/development/packages/structure.md b/doc/development/packages/structure.md new file mode 100644 index 00000000000..a2716232b11 --- /dev/null +++ b/doc/development/packages/structure.md @@ -0,0 +1,79 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Package Structure + +## Package Registry + +```mermaid +erDiagram + projects }|--|| namespaces : "" + packages_package_files }o--|| packages_packages : "" + packages_package_file_build_infos }o--|| packages_package_files : "" + packages_build_infos }o--|| packages_packages : "" + packages_tags }o--|| packages_packages : "" + packages_packages }|--|| projects : "" + packages_maven_metadata |o--|| packages_packages : "" + packages_nuget_metadata |o--|| packages_packages : "" + packages_composer_metadata |o--|| packages_packages : "" + packages_conan_metadata |o--|| packages_packages : "" + packages_pypi_metadata |o--|| packages_packages : "" + packages_npm_metadata |o--|| packages_packages : "" + package_conan_file_metadatum |o--|| packages_package_files : "" + package_helm_file_metadatum |o--|| packages_package_files : "" + packages_nuget_dependency_link_metadata |o--|| packages_dependency_links: "" + packages_dependencies ||--o| packages_dependency_links: "" + packages_packages ||--o{ packages_dependency_links: "" + namespace_package_settings |o--|| namespaces: "" +``` + +### Debian packages + +Debian contains a higher number of dedicated tables, so it is displayed here separately: + +```mermaid +erDiagram + projects }|--|| namespaces : "" + packages_packages }|--|| projects : "" + packages_package_files }o--|| packages_packages : "" + package_debian_file_metadatum |o--|| packages_package_files : "" + packages_debian_group_architectures }|--|| packages_debian_group_distributions : "" + packages_debian_group_component_files }|--|| packages_debian_group_components : "" + packages_debian_group_component_files }|--|| packages_debian_group_architectures : "" + packages_debian_group_components }|--|| packages_debian_group_distributions : "" + packages_debian_group_distribution_keys }|--|| packages_debian_group_distributions : "" + packages_debian_group_distributions }o--|| namespaces : "" + packages_debian_project_architectures }|--|| packages_debian_project_distributions : "" + packages_debian_project_component_files }|--|| packages_debian_project_components : "" + packages_debian_project_component_files }|--|| packages_debian_project_architectures : "" + packages_debian_project_components }|--|| packages_debian_project_distributions : "" + packages_debian_project_distribution_keys }|--|| packages_debian_project_distributions : "" + packages_debian_project_distributions }o--|| projects : "" + packages_debian_publications }|--|| packages_debian_project_distributions : "" + packages_debian_publications |o--|| packages_packages : "" + packages_debian_project_distributions |o--|| packages_packages : "" + packages_debian_group_distributions |o--|| namespaces : "" + packages_debian_file_metadata |o--|| packages_package_files : "" +``` + +## Container Registry + +```mermaid +erDiagram + projects }|--|| namespaces : "" + container_repositories }|--|| projects : "" + container_expiration_policy |o--|| projects : "" +``` + +## Dependency Proxy + +```mermaid +erDiagram + dependency_proxy_blobs }o--|| namespaces : "" + dependency_proxy_manifests }o--|| namespaces : "" + dependency_proxy_image_ttl_group_policies |o--|| namespaces : "" + dependency_proxy_group_settings |o--|| namespaces : "" +``` diff --git a/doc/development/pages/index.md b/doc/development/pages/index.md new file mode 100644 index 00000000000..02019db48ba --- /dev/null +++ b/doc/development/pages/index.md @@ -0,0 +1,238 @@ +--- +type: reference, dev +stage: Create +group: Editor +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +description: "GitLab's development guidelines for GitLab Pages" +--- + +# Getting started with development + +## Configuring GitLab Pages hostname + +GitLab Pages needs a hostname or domain, as each different GitLab Pages site is accessed via a +subdomain. GitLab Pages hostname can be set in different manners: + +- [Without wildcard, editing your hosts file](#without-wildcard-editing-your-hosts-file). +- [With DNS wildcard alternatives](#with-dns-wildcard-alternatives). + +### Without wildcard, editing your hosts file + +As `/etc/hosts` don't support wildcard hostnames, you must configure one entry +for GitLab Pages, and then one entry for each page site: + + ```plaintext + 127.0.0.1 gdk.test # If you're using GDK + 127.0.0.1 pages.gdk.test # Pages host + # Any namespace/group/user needs to be added + # as a subdomain to the pages host. This is because + # /etc/hosts doesn't accept wildcards + 127.0.0.1 root.pages.gdk.test # for the root pages + ``` + +### With DNS wildcard alternatives + +If instead of editing your `/etc/hosts` you'd prefer to use a DNS wildcard, you can use: + +- [`nip.io`](https://nip.io) +- [`dnsmasq`](https://wiki.debian.org/dnsmasq) + +## Configuring GitLab Pages without GDK + +Create a `gitlab-pages.conf` in the root of the GitLab Pages site, like: + +```toml +# Default port is 3010, but you can use any other +listen-http=:3010 + +# Your local GitLab Pages domain +pages-domain=pages.gdk.test + +# Directory where the pages are stored +pages-root=shared/pages + +# Show more information in the logs +log-verbose=true +``` + +To see more options you can check +[`internal/config/flags.go`](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/internal/config/flags.go) +or run `gitlab-pages --help`. + +### Running GitLab Pages manually + +For any changes in the code, you must run `make` to build the app. It's best to just always run +it before you start the app. It's quick to build so don't worry! + +```shell +make && ./gitlab-pages -config=gitlab-pages.conf +``` + +## Configuring GitLab Pages with GDK + +In the following steps, `$GDK_ROOT` is the directory where you cloned GDK. + +1. Set up the [GDK hostname](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/local_network.md). +1. Add a [GitLab Pages hostname](#configuring-gitlab-pages-hostname) to the `gdk.yml`: + + ```yaml + gitlab_pages: + enabled: true # enable GitLab Pages to be managed by gdk + port: 3010 # default port is 3010 + host: pages.gdk.test # the GitLab Pages domain + auto_update: true # if gdk must update GitLab Pages git + verbose: true # show more information in the logs + ``` + +### Running GitLab Pages with GDK + +After these configurations are set, GDK manages a GitLab Pages process, giving you access to +it with commands like: + +- Start: `gdk start gitlab-pages` +- Stop: `gdk stop gitlab-pages` +- Restart: `gdk restart gitlab-pages` +- Tail logs: `gdk tail gitlab-pages` + +### Running GitLab Pages manually + +You can also build and start the app independent of GDK processes management. + +For any changes in the code, you must run `make` to build the app. It's best to just always run +it before you start the app. It's quick to build so don't worry! + +```shell +make && ./gitlab-pages -config=gitlab-pages.conf +``` + +#### Building GitLab Pages in FIPS mode + +```shell +FIPS_MODE=1 make && ./gitlab-pages -config=gitlab-pages.conf +``` + +### Creating GitLab Pages site + +To build a GitLab Pages site locally you must +[configure `gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/runner.md) + +Check the [user manual](../../user/project/pages/index.md). + +### Enabling access control + +GitLab Pages support private sites. Private sites can be accessed only by users +who have access to your GitLab project. + +GitLab Pages access control is disabled by default. To enable it: + +1. Enable the GitLab Pages access control in GitLab itself, which can be done by either: + - If you're not using GDK, editing `gitlab.yml`: + + ```yaml + # gitlab/config/gitlab.yml + pages: + access_control: true + ``` + + - Editing `gdk.yml` if you're using GDK: + + ```yaml + # $GDK_ROOT/gdk.yml + gitlab_pages: + enabled: true + access_control: true + ``` + +1. Restart GitLab (if running through the GDK, run `gdk restart`). Running + `gdk reconfigure` overwrites the value of `access_control` in `config/gitlab.yml`. +1. In your local GitLab instance, in the browser go to `http://gdk.test:3000/admin/applications`. +1. Create an [Instance-wide OAuth application](../../integration/oauth_provider.md#instance-wide-applications) + with the `api` scope. +1. Set the value of your `redirect-uri` to the `pages-domain` authorization endpoint + - `http://pages.gdk.test:3010/auth`, for example + - The `redirect-uri` must not contain any GitLab Pages site domain. +1. Add the auth client configuration: + + - With GDK, in `gdk.yml`: + + ```yaml + gitlab_pages: + enabled: true + access_control: true + auth_client_id: $CLIENT_ID # the OAuth application id created in http://gdk.test:3000/admin/applications + auth_client_secret: $CLIENT_SECRET # the OAuth application secret created in http://gdk.test:3000/admin/applications + ``` + + GDK generates random `auth_secret` and builds the `auth_redirect_uri` based on GitLab Pages + host configuration. + + - Without GDK, in `gitlab-pages.conf`: + + ```conf + ## the following are only needed if you want to test auth for private projects + auth-client-id=$CLIENT_ID # the OAuth application id created in http://gdk.test:3000/admin/applications + auth-client-secret=$CLIENT_SECRET # the OAuth application secret created in http://gdk.test:3000/admin/applications + auth-secret=$SOME_RANDOM_STRING # should be at least 32 bytes long + auth-redirect-uri=http://pages.gdk.test:3010/auth # the authentication callback url for GitLab Pages + ``` + +1. If running Pages inside the GDK, you can use GDK's `protected_config_files` section under `gdk` in + your `gdk.yml` to avoid getting `gitlab-pages.conf` configuration rewritten: + + ```yaml + gdk: + protected_config_files: + - 'gitlab-pages/gitlab-pages.conf' + ``` + +### Enabling object storage + +GitLab Pages support using object storage for storing artifacts, but object storage +is disabled by default. You can enable it in the GDK: + +1. Edit `gdk.yml` to enable the object storage in GitLab itself: + + ```yaml + # $GDK_ROOT/gdk.yml + object_store: + enabled: true + ``` + +1. Reconfigure and restart GitLab by running the commands `gdk reconfigure` and `gdk restart`. + +For more information, refer to the [GDK documentation](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/configuration.md#object-storage-configuration). + +## Linting + +```shell +# Run the linter locally +make lint + +# Run linter and fix issues (if supported by the linter) +make format +``` + +## Testing + +To run tests, you can use these commands: + +```shell +# This will run all of the tests in the codebase +make test + +# Run a specfic test file +go test ./internal/serving/disk/ + +# Run a specific test in a file +go test ./internal/serving/disk/ -run TestDisk_ServeFileHTTP + +# Run all unit tests except acceptance_test.go +go test ./... -short + +# Run acceptance_test.go only +make acceptance +# Run specific acceptance tests +# We add `make` here because acceptance tests use the last binary that was compiled, +# so we want to have the latest changes in the build that is tested +make && go test ./ -run TestRedirect +``` diff --git a/doc/development/permissions.md b/doc/development/permissions.md index f3818e92fec..ed95456c4f9 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -95,11 +95,9 @@ can still view the groups and their entities (like epics). Project membership (where the group membership is already taken into account) is stored in the `project_authorizations` table. -WARNING: -Due to [an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299), -projects in personal namespace do not show owner (`50`) permission in -`project_authorizations` table. Note however that [`user.owned_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/0d63823b122b11abd2492bca47cc26858eee713d/app/models/user.rb#L906-916) -is calculated properly. +NOTE: +In [GitLab 14.9](https://gitlab.com/gitlab-org/gitlab/-/issues/351211) and later, projects in personal namespaces have a maximum role of Owner. +Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219299) in GitLab 14.8 and earlier, projects in personal namespaces have a maximum role of Maintainer. ### Confidential issues diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index b70f07ea7d9..436977a7f38 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -37,7 +37,7 @@ flowchart LR subgraph backend be["Backend code"]--tested with-->rspec end - + be--generates-->fixtures["frontend fixtures"] fixtures--used in-->jest ``` @@ -67,9 +67,9 @@ In the `detect-tests` job, we use this mapping to identify the minimal tests nee In addition, there are a few circumstances where we would always run the full RSpec tests: - when the `pipeline:run-all-rspec` label is set on the merge request -- when the merge request is created by an automation (e.g. Gitaly update or MR targeting a stable branch) +- when the merge request is created by an automation (for example, Gitaly update or MR targeting a stable branch) - when the merge request is created in a security mirror -- when any CI configuration file is changed (i.e. `.gitlab-ci.yml` or `.gitlab/ci/**/*`) +- when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) ### Jest minimal jobs @@ -83,11 +83,11 @@ In this mode, `jest` would resolve all the dependencies of related to the change In addition, there are a few circumstances where we would always run the full Jest tests: - when the `pipeline:run-all-jest` label is set on the merge request -- when the merge request is created by an automation (e.g. Gitaly update or MR targeting a stable branch) +- when the merge request is created by an automation (for example, Gitaly update or MR targeting a stable branch) - when the merge request is created in a security mirror -- when any CI configuration file is changed (i.e. `.gitlab-ci.yml` or `.gitlab/ci/**/*`) -- when any frontend "core" file is changed (i.e. `package.json`, `yarn.lock`, `babel.config.js`, `jest.config.*.js`, `config/helpers/**/*.js`) -- when any vendored JavaScript file is changed (i.e. `vendor/assets/javascripts/**/*`) +- when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) +- when any frontend "core" file is changed (for example, `package.json`, `yarn.lock`, `babel.config.js`, `jest.config.*.js`, `config/helpers/**/*.js`) +- when any vendored JavaScript file is changed (for example, `vendor/assets/javascripts/**/*`) - when any backend file is changed ([see the patterns list for details](https://gitlab.com/gitlab-org/gitlab/-/blob/3616946936c1adbd9e754c1bd06f86ba670796d8/.gitlab/ci/rules.gitlab-ci.yml#L205-216)) ### Fork pipelines @@ -97,6 +97,18 @@ label is set on the MR. The goal is to reduce the CI/CD minutes consumed by fork See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1170). +## Faster feedback when reverting merge requests + +When you need to revert a merge request, to get accelerated feedback, you can add the `~pipeline:revert` label to your merge request. + +When this label is assigned, the following steps of the CI/CD pipeline are skipped: + +- The `package-and-qa` job. +- The `rspec:undercoverage` job. +- The entire [Review Apps process](testing_guide/review_apps.md). + +Apply the label to the merge request, and run a new pipeline for the MR. + ## Fail-fast job in merge request pipelines To provide faster feedback when a merge request breaks existing tests, we are experimenting with a @@ -226,7 +238,7 @@ of `gitlab-org/gitlab-foss`. These jobs are only created in the following cases: - when the `pipeline:run-as-if-foss` label is set on the merge request - when the merge request is created in the `gitlab-org/security/gitlab` project -- when any CI configuration file is changed (i.e. `.gitlab-ci.yml` or `.gitlab/ci/**/*`) +- when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) The `* as-if-foss` jobs are run in addition to the regular EE-context jobs. They have the `FOSS_ONLY='1'` variable set and get the `ee/` folder removed before the tests start running. @@ -277,15 +289,23 @@ In the event of an emergency, or false positive from this job, add the `pipeline:skip-undercoverage` label to the merge request to allow this job to fail. -You can disable the `undercover` code coverage check by wrapping the desired block of code in `# :nocov:` lines: +### Troubleshooting `rspec:undercoverage` failures -```ruby -# :nocov: -def some_method - # code coverage for this method will be skipped -end -# :nocov: -``` +The `rspec:undercoverage` job has [known bugs](https://gitlab.com/groups/gitlab-org/-/epics/8254) +that can cause false positive failures. You can locally test coverage locally to determine if it's +safe to apply `~"pipeline:skip-undercoverage"`. For example, using `<spec>` as the name of the +test causing the failure: + +1. Run `SIMPLECOV=1 bundle exec rspec <spec>`. +1. Run `scripts/undercoverage`. + +If these commands return `undercover: ✅ No coverage is missing in latest changes` then you can apply `~"pipeline:skip-undercoverage"` to bypass pipeline failures. + +## Ruby versions testing + +Our test suite runs against Ruby 2 in merge requests and default branch pipelines. + +We do run our test suite against Ruby 3 on 2-hourly scheduled pipelines, as GitLab.com will soon run on Ruby 3. ## PostgreSQL versions testing @@ -339,7 +359,7 @@ In general, pipelines for an MR fall into one of the following types (from short - [Frontend pipeline](#frontend-pipeline): For MRs that touch frontend code. - [End-to-end pipeline](#end-to-end-pipeline): For MRs that touch code in the `qa/` folder. -A "pipeline type" is an abstract term that mostly describes the "critical path" (i.e. the chain of jobs for which the sum +A "pipeline type" is an abstract term that mostly describes the "critical path" (for example, the chain of jobs for which the sum of individual duration equals the pipeline's duration). We use these "pipeline types" in [metrics dashboards](https://app.periscopedata.com/app/gitlab/858266/GitLab-Pipeline-Durations) in order to detect what types and jobs need to be optimized first. @@ -719,11 +739,11 @@ This job tries to download a generic package that contains GitLab Workhorse bina We also changed the `setup-test-env` job to: 1. First download the GitLab Workhorse generic package build and uploaded by `build-components`. -1. If the package is retrieved successfully, its content is placed in the right folder (i.e. `tmp/tests/gitlab-workhorse`), preventing the building of the binaries when `scripts/setup-test-env` is run later on. +1. If the package is retrieved successfully, its content is placed in the right folder (for example, `tmp/tests/gitlab-workhorse`), preventing the building of the binaries when `scripts/setup-test-env` is run later on. 1. If the package URL returns a 404, the behavior doesn't change compared to the current one: the GitLab Workhorse binaries are built as part of `scripts/setup-test-env`. NOTE: -The version of the package is the workhorse tree SHA (i.e. `git rev-parse HEAD:workhorse`). +The version of the package is the workhorse tree SHA (for example, `git rev-parse HEAD:workhorse`). ### Pre-clone step diff --git a/doc/development/product_qualified_lead_guide/index.md b/doc/development/product_qualified_lead_guide/index.md index dcd8b33e5c5..25634876aef 100644 --- a/doc/development/product_qualified_lead_guide/index.md +++ b/doc/development/product_qualified_lead_guide/index.md @@ -21,7 +21,7 @@ A hand-raise PQL is a user who requests to speak to sales from within the produc 1. Enter the credentials on CustomersDot development to Platypus in your `/config/secrets.yml` and restart. Credentials for the Platypus Staging are in the 1Password Growth vault. The URL for staging is `https://staging.ci.nexus.gitlabenvironment.cloud`. ```yaml - platypus_url: "<%= ENV['PLATYPUS_URL'] %>" + platypus_url: "<%= ENV['PLATYPUS_URL'] %>" platypus_client_id: "<%= ENV['PLATYPUS_CLIENT_ID'] %>" platypus_client_secret: "<%= ENV['PLATYPUS_CLIENT_SECRET'] %>" ``` @@ -42,7 +42,7 @@ A hand-raise PQL is a user who requests to speak to sales from within the produc - Check the application and Sidekiq logs on `gitlab.com` and CustomersDot to monitor leads. - Check the `leads` table in CustomersDot. -- Set up staging credentials for Platypus, and track the leads on the [Platypus Dashboard](https://staging.ci.nexus.gitlabenvironment.cloud/admin/queues/queue/new-lead-queue). +- Set up staging credentials for Platypus, and track the leads on the Platypus Dashboard: `https://staging.ci.nexus.gitlabenvironment.cloud/admin/queues/queue/new-lead-queue`. - Ask for access to the Marketo Sandbox and validate the leads there, [to this example request](https://gitlab.com/gitlab-com/team-member-epics/access-requests/-/issues/13162). ## Embed a hand-raise lead form diff --git a/doc/development/project_templates.md b/doc/development/project_templates.md index 74ded9c93fc..f688d54ad4f 100644 --- a/doc/development/project_templates.md +++ b/doc/development/project_templates.md @@ -4,59 +4,56 @@ group: Workspace info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" --- -# Contribute to GitLab project templates +# Contribute a built-in project template -Thanks for considering a contribution to the GitLab -[built-in project templates](../user/project/working_with_projects.md#create-a-project-from-a-built-in-template). +This page provides instructions about how to contribute a +[built-in project template](../user/project/working_with_projects.md#create-a-project-from-a-built-in-template). + +To contribute a built-in project template, you must complete the following tasks: + +1. [Create a project template for GitLab review](#create-a-project-template-for-review) +1. [Add the template SVG icon to GitLab SVGs](#add-the-template-svg-icon-to-gitlab-svgs) +1. [Create a merge request with vendor details](#create-a-merge-request-with-vendor-details) + +You can contribute the following types of project templates: + +- Enterprise: For users with GitLab Premium and above. +- Non-enterprise: For users with GitLab Free and above. ## Prerequisites -To add a new or update an existing template, you must have the following tools +To add or update an existing template, you must have the following tools installed: - `wget` - `tar` -- `jq` - -## Create a new project -To contribute a new built-in project template to be distributed with GitLab: +## Create a project template for review -1. Create a new public project with the project content you'd like to contribute - in a namespace of your choosing. You can [view a working example](https://gitlab.com/gitlab-org/project-templates/dotnetcore). - Projects should be as simple as possible and free of any unnecessary assets or dependencies. -1. When the project is ready for review, [create a new issue](https://gitlab.com/gitlab-org/gitlab/issues) with a link to your project. - In your issue, `@` mention the relevant Backend Engineering Manager and Product - Manager for the [Templates feature](https://about.gitlab.com/handbook/product/categories/#source-code-group). +1. In your selected namespace, create a public project. +1. Add the project content you want to use in the template. Do not include unnecessary assets or dependencies. For an example, +[see this project](https://gitlab.com/gitlab-org/project-templates/dotnetcore). +1. When the project is ready for review, [create an issue](https://gitlab.com/gitlab-org/gitlab/issues) with a link to your project. + In your issue, mention the relevant [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) + for the Templates feature. -## Add the SVG icon to GitLab SVGs +## Add the template SVG icon to GitLab SVGs -If the template you're adding has an SVG icon, you need to first add it to -<https://gitlab.com/gitlab-org/gitlab-svgs>: +If the project template has an SVG icon, you must add it to the +[GitLab SVGs project](https://gitlab.com/gitlab-org/gitlab-svgs/-/blob/main/README.md#adding-icons-or-illustrations) +before you can create a merge request with vendor details. -1. Follow the steps outlined in the - [GitLab SVGs project](https://gitlab.com/gitlab-org/gitlab-svgs/-/blob/main/README.md#adding-icons-or-illustrations) - and submit a merge request. -1. When the merge request is merged, `gitlab-bot` will pull the new changes in - the `gitlab-org/gitlab` project. -1. You can now continue on the vendoring process. +## Create a merge request with vendor details -## Vendoring process - -To make the project template available when creating a new project, the vendoring -process will have to be completed: +Before GitLab can implement the project template, you must [create a merge request](../user/project/merge_requests/creating_merge_requests.md) in [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab) that includes vendor details about the project. 1. [Export the project](../user/project/settings/import_export.md#export-a-project-and-its-data) - you created in the previous step and save the file as `<name>.tar.gz`, where - `<name>` is the short name of the project. -1. Edit the following files to include the project template. Two types of built-in - templates are available within GitLab: - - **Normal templates**: Available in GitLab Free and above (this is the most common type of built-in template). - See MR [!25318](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25318) for an example. - - To add a normal template: - - 1. Open `lib/gitlab/project_template.rb` and add details of the template + and save the file as `<name>.tar.gz`, where `<name>` is the short name of the project. + Move this file to the root directory of `gitlab-org/gitlab`. +1. In `gitlab-org/gitlab`, create and checkout a new branch. +1. Edit the following files to include the project template: + - For **non-Enterprise** project templates: + - In `lib/gitlab/project_template.rb`, add details about the template in the `localized_templates_table` method. In the following example, the short name of the project is `hugo`: @@ -64,11 +61,11 @@ process will have to be completed: ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), ``` - If the vendored project doesn't have an SVG icon, omit `, 'illustrations/logos/hugo.svg'`. + If the project doesn't have an SVG icon, exclude `, 'illustrations/logos/hugo.svg'`. - 1. Open `spec/lib/gitlab/project_template_spec.rb` and add the short name - of the template in the `.all` test. - 1. Open `app/assets/javascripts/projects/default_project_templates.js` and + - In `spec/support/helpers/project_template_test_helper.rb`, append the short name + of the template in the `all_templates` method. + - In `app/assets/javascripts/projects/default_project_templates.js`, add details of the template. For example: ```javascript @@ -78,25 +75,19 @@ process will have to be completed: }, ``` - If the vendored project doesn't have an SVG icon, use `.icon-gitlab_logo` + If the project doesn't have an SVG icon, use `.icon-gitlab_logo` instead. - - - **Enterprise templates**: Introduced in GitLab 12.10, that are available only in GitLab Premium and above. - See MR [!28187](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28187) for an example. - - To add an Enterprise template: - - 1. Open `ee/lib/ee/gitlab/project_template.rb` and add details of the template - in the `localized_ee_templates_table` method. For example: + - For **Enterprise** project templates: + - In `ee/lib/ee/gitlab/project_template.rb`, in the `localized_ee_templates_table` method, add details about the template. For example: ```ruby ::Gitlab::ProjectTemplate.new('hipaa_audit_protocol', 'HIPAA Audit Protocol', _('A project containing issues for each audit inquiry in the HIPAA Audit Protocol published by the U.S. Department of Health & Human Services'), 'https://gitlab.com/gitlab-org/project-templates/hipaa-audit-protocol', 'illustrations/logos/asklepian.svg') ``` - 1. Open `ee/spec/lib/gitlab/project_template_spec.rb` and add the short name + - In `ee/spec/lib/gitlab/project_template_spec.rb`, add the short name of the template in the `.all` test. - 1. Open `ee/app/assets/javascripts/projects/default_project_templates.js` and - add details of the template. For example: + - In `ee/app/assets/javascripts/projects/default_project_templates.js`, + add the template details. For example: ```javascript hipaa_audit_protocol: { @@ -105,10 +96,11 @@ process will have to be completed: }, ``` -1. Run the `vendor_template` script. Make sure to pass the correct arguments: +1. Run the following Rake task, where `<path>/<name>` is the + name you gave the template in `lib/gitlab/project_template.rb`: ```shell - scripts/vendor_template <git_repo_url> <name> <comment> + bin/rake gitlab:update_project_templates\[<path>/<name>\] ``` 1. Regenerate `gitlab.pot`: @@ -117,41 +109,24 @@ process will have to be completed: bin/rake gettext:regenerate ``` -1. By now, there should be one new file under `vendor/project_templates/` and - 4 changed files. Commit all of them in a new branch and create a merge - request. +1. After you run the scripts, there is one new file in `vendor/project_templates/` and four changed files. Commit all changes and push your branch to update the merge request. For an example, see this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25318). -## Test with GDK +## Test your built-in project with the GitLab Development Kit -If you are using the GitLab Development Kit (GDK) you must disable `praefect` -and regenerate the Procfile, as the Rake task is not currently compatible with it: +Complete the following steps to test the project template in your own GitLab Development Kit instance: -```yaml -# gitlab-development-kit/gdk.yml -praefect: - enabled: false -``` - -1. Follow the steps described in the [vendoring process](#vendoring-process). -1. Run the following Rake task where `<path>/<name>` is the +1. Run the following Rake task, where `<path>/<name>` is the name you gave the template in `lib/gitlab/project_template.rb`: ```shell - bin/rake gitlab:update_project_templates[<path>/<name>] + bin/rake gitlab:update_project_templates\[<path>/<name>\] ``` -You can now test to create a new project by importing the new template in GDK. - ## Contribute an improvement to an existing template -Existing templates are imported from the following groups: - -- [`project-templates`](https://gitlab.com/gitlab-org/project-templates) -- [`pages`](htps://gitlab.com/pages) - -To contribute a change, open a merge request in the relevant project -and mention `@gitlab-org/manage/import/backend` when you are ready for a review. +To update an existing built-in project template: -Then, if your merge request gets accepted, either [open an issue](https://gitlab.com/gitlab-org/gitlab/-/issues) -to ask for it to get updated, or open a merge request updating -the [vendored template](#vendoring-process). +1. Create a merge request in the relevant project of the `project-templates` and `pages` group and mention `@gitlab-org/manage/import/backend` when you are ready for a review. +1. If your merge request is accepted, either: + - [Create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues) to ask for the template to get updated. + - [Create a merge request with vendor details](#create-a-merge-request-with-vendor-details) to update the template. diff --git a/doc/development/query_count_limits.md b/doc/development/query_count_limits.md index fec6f9022ee..49509727337 100644 --- a/doc/development/query_count_limits.md +++ b/doc/development/query_count_limits.md @@ -58,7 +58,7 @@ By using a `before_action` you don't have to modify the controller method in question, reducing the likelihood of merge conflicts. For Grape API endpoints there unfortunately is not a reliable way of running a -hook before a specific endpoint. This means that you have to add the whitelist +hook before a specific endpoint. This means that you have to add the allowlist call directly into the endpoint like so: ```ruby diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md index bc1f753c012..4fe27d42c38 100644 --- a/doc/development/query_performance.md +++ b/doc/development/query_performance.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -11,7 +11,7 @@ This document describes various guidelines to follow when optimizing SQL queries When you are optimizing your SQL queries, there are two dimensions to pay attention to: 1. The query execution time. This is paramount as it reflects how the user experiences GitLab. -1. The query plan. Optimizing the query plan is important in allowing queries to independently scale over time. Realizing that an index will keep a query performing well as the table grows before the query degrades is an example of why we analyze these plans. +1. The query plan. Optimizing the query plan is important in allowing queries to independently scale over time. Realizing that an index keeps a query performing well as the table grows before the query degrades is an example of why we analyze these plans. ## Timing guidelines for queries @@ -39,9 +39,9 @@ cache, or what PostgreSQL calls shared buffers. This is the "warm cache" query. When analyzing an [`EXPLAIN` plan](understanding_explain_plans.md), you can see the difference not only in the timing, but by looking at the output for `Buffers` by running your explain with `EXPLAIN(analyze, buffers)`. [Database Lab](understanding_explain_plans.md#database-lab-engine) -will automatically include these options. +automatically includes these options. -If you are making a warm cache query, you will only see the `shared hits`. +If you are making a warm cache query, you see only the `shared hits`. For example in #database-lab: @@ -57,7 +57,7 @@ Or in the explain plan from `psql`: Buffers: shared hit=7323 ``` -If the cache is cold, you will also see `reads`. +If the cache is cold, you also see `reads`. In #database-lab: diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index 17f2fecc1bc..371d6e0e49e 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -24,10 +24,27 @@ it "avoids N+1 database queries" do end ``` +You can if you wish, have both the expectation and the control as +`QueryRecorder` instances: + +```ruby +it "avoids N+1 database queries" do + control = ActiveRecord::QueryRecorder.new { visit_some_page } + create_list(:issue, 5) + action = ActiveRecord::QueryRecorder.new { visit_some_page } + + expect(action).not_to exceed_query_limit(control) +end +``` + As an example you might create 5 issues in between counts, which would cause the query count to increase by 5 if an N+1 problem exists. In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. +If this test fails, and the control was passed as a `QueryRecorder`, then the +failure message indicates where the extra queries are by matching queries on +the longest common prefix, grouping similar queries together. + ## Cached queries By default, QueryRecorder ignores [cached queries](merge_request_performance_guidelines.md#cached-queries) in the count. However, it may be better to count diff --git a/doc/development/rails_initializers.md b/doc/development/rails_initializers.md index 9bf4109f1cb..68f3c07e45a 100644 --- a/doc/development/rails_initializers.md +++ b/doc/development/rails_initializers.md @@ -24,13 +24,13 @@ Some examples where you would need to do this are: ## Database connections in initializers Ideally, database connections are not opened from Rails initializers. Opening a -database connection (e.g. checking the database exists, or making a database +database connection (for example, checking the database exists, or making a database query) from an initializer means that tasks like `db:drop`, and `db:test:prepare` will fail because an active session prevents the database from being dropped. To help detect when database connections are opened from initializers, we now -warn in stderr. For example: +warn in `STDERR`. For example: ```shell DEPRECATION WARNING: Database connection should not be called during initializers (called from block in <module:HasVariable> at app/models/concerns/ci/has_variable.rb:22) diff --git a/doc/development/rails_update.md b/doc/development/rails_update.md index 8999ac90f4c..36ffae97377 100644 --- a/doc/development/rails_update.md +++ b/doc/development/rails_update.md @@ -24,6 +24,7 @@ We strive to run GitLab using the latest Rails releases to benefit from performa 1. Run `bundle update --conservative activesupport` in the `qa` folder. 1. Resolve any Bundler conflicts. 1. Ensure that `@rails/ujs` and `@rails/actioncable` npm packages match the new rails version in [`package.json`](https://gitlab.com/gitlab-org/gitlab/blob/master/package.json). +1. Run `yarn patch-package @rails/ujs` after updating this to ensure our local patch file version matches. 1. Create an MR with the `pipeline:run-all-rspec` label and see if pipeline breaks. 1. To resolve and debug spec failures use `git bisect` against the rails repository. See the [debugging section](#git-bisect-against-rails) below. 1. Include links to the Gem diffs between the two versions in the merge request description. For example, this is the gem diff for [`activesupport` 6.1.3.2 to diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index 0538add59b5..13c4bdaedca 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -166,7 +166,7 @@ There are a few caveats for this Rake task: - The pipeline must have been completed. - You may need to wait for the test report to be parsed and retry again. -This Rake task depends on the [unit test reports](../ci/unit_test_reports.md) feature, +This Rake task depends on the [unit test reports](../ci/testing/unit_test_reports.md) feature, which only gets parsed when it is requested for the first time. ### Speed up tests, Rake tasks, and migrations diff --git a/doc/development/redis.md b/doc/development/redis.md index d5f526f2d32..e48048be624 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -56,8 +56,7 @@ the entry, instead of relying on the key changing. ### Multi-key commands -We don't use [Redis Cluster](https://redis.io/topics/cluster-tutorial) at the -moment, but may wish to in the future: [#118820](https://gitlab.com/gitlab-org/gitlab/-/issues/118820). +We don't use Redis Cluster, but support for it is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/118820). This imposes an additional constraint on naming: where GitLab is performing operations that require several keys to be held on the same Redis server - for @@ -118,12 +117,15 @@ NOTE: There is a [video showing how to see the slow log](https://youtu.be/BBI68QuYRH8) (GitLab internal) on GitLab.com -On GitLab.com, entries from the [Redis -slow log](https://redis.io/commands/slowlog) are available in the +<!-- vale gitlab.Substitutions = NO --> + +On GitLab.com, entries from the [Redis slow log](https://redis.io/commands/slowlog/) are available in the `pubsub-redis-inf-gprd*` index with the [`redis.slowlog` tag](https://log.gprd.gitlab.net/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&_a=(columns:!(json.type,json.command,json.exec_time_s),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:AWSQX_Vf93rHTYrsexmk,key:json.tag,negate:!f,params:(query:redis.slowlog),type:phrase),query:(match:(json.tag:(query:redis.slowlog,type:phrase))))),index:AWSQX_Vf93rHTYrsexmk)). This shows commands that have taken a long time and may be a performance concern. +<!-- vale gitlab.Substitutions = YES --> + The [`fluent-plugin-redis-slowlog`](https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog) project is responsible for taking the `slowlog` entries from Redis and @@ -183,9 +185,9 @@ makes sure that booleans are encoded and decoded consistently. ### `Gitlab::Redis::HLL` -The Redis [`PFCOUNT`](https://redis.io/commands/pfcount), -[`PFADD`](https://redis.io/commands/pfadd), and -[`PFMERGE`](https://redis.io/commands/pfmergge) commands operate on +The Redis [`PFCOUNT`](https://redis.io/commands/pfcount/), +[`PFADD`](https://redis.io/commands/pfadd/), and +[`PFMERGE`](https://redis.io/commands/pfmerge/) commands operate on HyperLogLogs, a data structure that allows estimating the number of unique elements with low memory usage. (In addition to the `PFCOUNT` documentation, Thoughtbot's article on [HyperLogLogs in Redis](https://thoughtbot.com/blog/hyperloglogs-in-redis) @@ -200,7 +202,7 @@ For cases where we need to efficiently check the whether an item is in a group of items, we can use a Redis set. [`Gitlab::SetCache`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/set_cache.rb) provides an `#include?` method that uses the -[`SISMEMBER`](https://redis.io/commands/sismember) command, as well as `#read` +[`SISMEMBER`](https://redis.io/commands/sismember/) command, as well as `#read` to fetch all entries in the set. This is used by the diff --git a/doc/development/redis/new_redis_instance.md b/doc/development/redis/new_redis_instance.md index 389cddbb4e5..4900755b58c 100644 --- a/doc/development/redis/new_redis_instance.md +++ b/doc/development/redis/new_redis_instance.md @@ -179,9 +179,12 @@ bin/feature-flag use_primary_store_as_default_for_foo By enabling `use_primary_and_secondary_stores_for_foo` feature flag, our `Gitlab::Redis::Foo` will use `MultiStore` to write to both new Redis instance and the [old (fallback-instance)](#fallback-instance). If we fail to fetch data from the new instance, we will fallback and read from the old Redis instance. - We can monitor logs for `Gitlab::Redis::MultiStore::ReadFromPrimaryError`, and also the Prometheus counter `gitlab_redis_multi_store_read_fallback_total`. -Once we stop seeing them, this means that we are no longer relying on the data stored on the old Redis store. + +For pipelined commands (`pipelined` and `multi`), we execute the entire operation in both stores and then compare the results. If they differ, we emit a +`Gitlab::Redis::MultiStore:PipelinedDiffError` error, and track it in the `gitlab_redis_multi_store_pipelined_diff_error_total` Prometheus counter. + +Once we stop seeing those errors, this means that we are no longer relying on the data stored on the old Redis store. At this point, we are probably safe to move the traffic to the new Redis store. By enabling `use_primary_store_as_default_for_foo` feature flag, the `MultiStore` will use `primary_store` (new instance) as default Redis store. @@ -213,6 +216,15 @@ MultiStore implements read and write Redis commands separately. - `del` - `pipelined` - `flushdb` +- `rpush` + +##### Pipelined commands + +**NOTE:** The Ruby block passed to these commands will be executed twice, once per each store. +Thus, excluding the Redis operations performed, the block should be idempotent. + +- `pipelined` +- `multi` When a command outside of the supported list is used, `method_missing` will pass it to the old Redis instance and keep track of it. This ensures that anything unexpected behaves like it would before. @@ -223,17 +235,19 @@ a developer will need to add an implementation for missing Redis commands before ##### Errors -| error | message | -|-------------------------------------------------|-----------------------------------------------------------------------| +| error | message | +|---------------------------------------------------|---------------------------------------------------------------------------------------------| | `Gitlab::Redis::MultiStore::ReadFromPrimaryError` | Value not found on the Redis primary store. Read from the Redis secondary store successful. | -| `Gitlab::Redis::MultiStore::MethodMissingError` | Method missing. Falling back to execute method on the Redis secondary store. | +| `Gitlab::Redis::MultiStore::PipelinedDiffError` | Pipelined command executed on both stores successfully but results differ between them. | +| `Gitlab::Redis::MultiStore::MethodMissingError` | Method missing. Falling back to execute method on the Redis secondary store. | ##### Metrics -| metrics name | type | labels | description | -|-------------------------------------------------|--------------------|------------------------|----------------------------------------------------| -| `gitlab_redis_multi_store_read_fallback_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore reading fallback total| -| `gitlab_redis_multi_store_method_missing_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore method missing total | +| metrics name | type | labels | description | +|-------------------------------------------------------|--------------------|------------------------|--------------------------------------------------------| +| `gitlab_redis_multi_store_read_fallback_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore reading fallback total | +| `gitlab_redis_multi_store_pipelined_diff_error_total` | Prometheus Counter | command, instance_name | Redis MultiStore pipelined command diff between stores | +| `gitlab_redis_multi_store_method_missing_total` | Prometheus Counter | command, instance_name | Client side Redis MultiStore method missing total | ## Step 4: clean up after the migration diff --git a/doc/development/refactoring_guide/index.md b/doc/development/refactoring_guide/index.md index a6ed83258f3..9793db3bb85 100644 --- a/doc/development/refactoring_guide/index.md +++ b/doc/development/refactoring_guide/index.md @@ -71,7 +71,7 @@ expect(cleanForSnapshot(wrapper.element)).toMatchSnapshot(); ### Resources -[Unofficial wiki explanation](http://wiki.c2.com/?PinningTests) +[Unofficial wiki explanation](https://wiki.c2.com/?PinningTests) ### Examples diff --git a/doc/development/reference_processing.md b/doc/development/reference_processing.md index ad6552e88fe..1dfe6496e79 100644 --- a/doc/development/reference_processing.md +++ b/doc/development/reference_processing.md @@ -41,7 +41,7 @@ For example, the class is responsible for handling references to issues, such as `gitlab-org/gitlab#123` and `https://gitlab.com/gitlab-org/gitlab/-/issues/200048`. -All reference filters are instances of [`HTML::Pipeline::Filter`](https://www.rubydoc.info/github/jch/html-pipeline/HTML/Pipeline/Filter), +All reference filters are instances of [`HTML::Pipeline::Filter`](https://www.rubydoc.info/gems/html-pipeline), and inherit (often indirectly) from [`Banzai::Filter::ReferenceFilter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/banzai/filter/reference_filter.rb). `HTML::Pipeline::Filter` has a simple interface consisting of `#call`, a void diff --git a/doc/development/routing.md b/doc/development/routing.md index 41961c2288f..2b3ecd8127b 100644 --- a/doc/development/routing.md +++ b/doc/development/routing.md @@ -104,6 +104,6 @@ To get started, see an [example merge request](https://gitlab.com/gitlab-org/git ## Useful links -- [Routing improvements master plan](https://gitlab.com/gitlab-org/gitlab/-/issues/215362) +- [Routing improvements main plan](https://gitlab.com/gitlab-org/gitlab/-/issues/215362) - [Scoped routing explained](https://gitlab.com/gitlab-org/gitlab/-/issues/214217) - [Removal of deprecated routes](https://gitlab.com/gitlab-org/gitlab/-/issues/28848) diff --git a/doc/development/ruby3_gotchas.md b/doc/development/ruby3_gotchas.md index e4ed5039e3c..dbe6fa13eee 100644 --- a/doc/development/ruby3_gotchas.md +++ b/doc/development/ruby3_gotchas.md @@ -138,3 +138,28 @@ installed Ruby manually or via tools like `asdf`. Users of the `gitlab-developme are also affected by this problem. Build images are not affected because they include the patch set addressing this bug. + +## Deprecations are not caught in DeprecationToolkit if the method is stubbed + +We rely on `deprecation_toolkit` to fail fast when using functionality that is deprecated in Ruby 2 and removed in Ruby 3. +A common issue caught during the transition from Ruby 2 to Ruby 3 relates to +the [separation of positional and keyword arguments in Ruby 3.0](https://www.ruby-lang.org/en/news/2019/12/12/separation-of-positional-and-keyword-arguments-in-ruby-3-0/). + +Unfortunately, if the author has stubbed such methods in tests, deprecations would not be caught. +We run automated detection for this warning in tests via `deprecation_toolkit`, +but it relies on the fact that `Kernel#warn` emits a warning, so stubbing out this call will effectively remove the call to warn, which means `deprecation_toolkit` will never see the deprecation warnings. +Stubbing out the implementation removes that warning, and we never pick it up, so the build is green. + +Please refer to [issue 364099](https://gitlab.com/gitlab-org/gitlab/-/issues/364099) for more context. + +## Testing in `irb` and `rails console` + +Another pitfall is that testing in `irb`/`rails c` silences the deprecation warning, +since `irb` in Ruby 2.7.x has a [bug](https://bugs.ruby-lang.org/issues/17377) that prevents deprecation warnings from showing. + +When writing code and performing code reviews, pay extra attention to method calls of the form `f({k: v})`. +This is valid in Ruby 2 when `f` takes either a `Hash` or keyword arguments, but Ruby 3 only considers this valid if `f` takes a `Hash`. +For Ruby 3 compliance, this should be changed to one of the following invocations if `f` takes keyword arguments: + +- `f(**{k: v})` +- `f(k: v)` diff --git a/doc/development/scalability.md b/doc/development/scalability.md index 4450df0399d..39cd0ecfcdd 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -222,7 +222,7 @@ only when the primary fails. ### Redis Sentinels -[Redis Sentinel](https://redis.io/topics/sentinel) provides high +[Redis Sentinel](https://redis.io/docs/manual/sentinel/) provides high availability for Redis by watching the primary. If multiple Sentinels detect that the primary has gone away, the Sentinels performs an election to determine a new leader. @@ -232,8 +232,7 @@ election to determine a new leader. No leader: A Redis cluster can get into a mode where there are no primaries. For example, this can happen if Redis nodes are misconfigured to follow the wrong node. Sometimes this requires forcing one node to -become a primary by using the [`REPLICAOF NO ONE` -command](https://redis.io/commands/replicaof). +become a primary by using the [`REPLICAOF NO ONE` command](https://redis.io/commands/replicaof/). ### Sidekiq @@ -275,8 +274,8 @@ in a timely manner: this to `ProcessCommitWorker`. - Redistribute/gerrymander Sidekiq processes by queue types. Long-running jobs (for example, relating to project import) can often - squeeze out jobs that run fast (for example, delivering email). [This technique - was used in to optimize our existing Sidekiq deployment](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7219#note_218019483). + squeeze out jobs that run fast (for example, delivering email). + [We used this technique to optimize our existing Sidekiq deployment](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/7219#note_218019483). - Optimize jobs. Eliminating unnecessary work, reducing network calls (including SQL and Gitaly), and optimizing processor time can yield significant benefits. diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 3e46891d20e..d8e2352bd93 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -59,7 +59,7 @@ Some example of well implemented access controls and tests: 1. [example2](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/2511/diffs#ed3aaab1510f43b032ce345909a887e5b167e196_142_155) 1. [example3](https://dev.gitlab.org/gitlab/gitlabhq/-/merge_requests/3170/diffs?diff_id=17494) -**NB:** any input from development team is welcome, for example, about Rubocop rules. +**NB:** any input from development team is welcome, for example, about RuboCop rules. ## Regular Expressions guidelines @@ -637,14 +637,11 @@ We recommend using the ciphers that Mozilla is providing in their [recommended S - `ECDHE-RSA-AES128-GCM-SHA256` - `ECDHE-ECDSA-AES256-GCM-SHA384` - `ECDHE-RSA-AES256-GCM-SHA384` -- `ECDHE-ECDSA-CHACHA20-POLY1305` -- `ECDHE-RSA-CHACHA20-POLY1305` And the following cipher suites (according to the [RFC 8446](https://datatracker.ietf.org/doc/html/rfc8446#appendix-B.4)) for TLS 1.3: - `TLS_AES_128_GCM_SHA256` - `TLS_AES_256_GCM_SHA384` -- `TLS_CHACHA20_POLY1305_SHA256` *Note*: **Golang** does [not support](https://github.com/golang/go/blob/go1.17/src/crypto/tls/cipher_suites.go#L676) all cipher suites with TLS 1.3. @@ -665,7 +662,7 @@ For **Ruby**, you can use [`HTTParty`](https://github.com/jnunemaker/httparty) a Whenever possible this example should be **avoided** for security purposes: ```ruby -response = HTTParty.get('https://gitlab.com', ssl_version: :TLSv1_3, ciphers: ['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256']) +response = HTTParty.get('https://gitlab.com', ssl_version: :TLSv1_3, ciphers: ['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384']) ``` When using [`GitLab::HTTP`](#gitlab-http-library), the code looks like: @@ -673,7 +670,7 @@ When using [`GitLab::HTTP`](#gitlab-http-library), the code looks like: This is the **recommended** implementation to avoid security issues such as SSRF: ```ruby -response = GitLab::HTTP.perform_request(Net::HTTP::Get, 'https://gitlab.com', ssl_version: :TLSv1_3, ciphers: ['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256']) +response = GitLab::HTTP.perform_request(Net::HTTP::Get, 'https://gitlab.com', ssl_version: :TLSv1_3, ciphers: ['TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384']) ``` ##### TLS 1.2 @@ -687,8 +684,6 @@ func secureCipherSuites() []uint16 { tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, } ``` @@ -703,12 +698,12 @@ tls.Config{ } ``` -This example was taken [here](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/871b52dc700f1a66f6644fbb1e78a6d463a6ff83/internal/tool/tlstool/tlstool.go#L72). +This example was taken [from the GitLab Agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/871b52dc700f1a66f6644fbb1e78a6d463a6ff83/internal/tool/tlstool/tlstool.go#L72). For **Ruby**, you can use again [`HTTParty`](https://github.com/jnunemaker/httparty) and specify this time TLS 1.2 version alongside with the recommended ciphers: ```ruby -response = GitLab::HTTP.perform_request(Net::HTTP::Get, 'https://gitlab.com', ssl_version: :TLSv1_2, ciphers: ['ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-CHACHA20-POLY1305', 'ECDHE-RSA-CHACHA20-POLY1305']) +response = GitLab::HTTP.perform_request(Net::HTTP::Get, 'https://gitlab.com', ssl_version: :TLSv1_2, ciphers: ['ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384']) ``` ## GitLab Internal Authorization diff --git a/doc/development/serializing_data.md b/doc/development/serializing_data.md index 48e756d015b..97e6f665484 100644 --- a/doc/development/serializing_data.md +++ b/doc/development/serializing_data.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -20,7 +20,7 @@ end ``` While it may be tempting to store serialized data in the database there are many -problems with this. This document will outline these problems and provide an +problems with this. This document outlines these problems and provide an alternative. ## Serialized Data Is Less Powerful @@ -34,13 +34,13 @@ turn there's no way to query the data at all. ## Waste Of Space -Storing serialized data such as JSON or YAML will end up wasting a lot of space. +Storing serialized data such as JSON or YAML ends up wasting a lot of space. This is because these formats often include additional characters (for example, double quotes or newlines) besides the data that you are storing. ## Difficult To Manage -There comes a time where you will need to add a new field to the serialized +There comes a time where you must add a new field to the serialized data, or change an existing one. Using serialized data this becomes difficult and very time consuming as the only way of doing so is to re-write all the stored values. To do so you would have to: @@ -51,8 +51,7 @@ stored values. To do so you would have to: 1. Serialize it back to a String 1. Store it in the database -On the other hand, if one were to use regular columns adding a column would be -as easy as this: +On the other hand, if one were to use regular columns adding a column would be: ```sql ALTER TABLE table_name ADD COLUMN column_name type; @@ -62,9 +61,9 @@ Such a query would take very little to no time and would immediately apply to all rows, without having to re-write large JSON or YAML structures. Finally, there comes a time when the JSON or YAML structure is no longer -sufficient and you need to migrate away from it. When storing only a few rows +sufficient and you must migrate away from it. When storing only a few rows this may not be a problem, but when storing millions of rows such a migration -can easily take hours or even days to complete. +can take hours or even days to complete. ## Relational Databases Are Not Document Stores @@ -85,7 +84,7 @@ you don't need. ## The Solution -The solution is very simple: just use separate columns and/or separate tables. -This will allow you to use all the features provided by your database, it will -make it easier to manage and migrate the data, you'll conserve space, you can +The solution is to use separate columns and/or separate tables. +This allows you to use all the features provided by your database, it +makes it easier to manage and migrate the data, you conserve space, you can index the data efficiently and so forth. diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md index 27bc4d2e8ca..6948eb20e00 100644 --- a/doc/development/service_ping/implement.md +++ b/doc/development/service_ping/implement.md @@ -46,7 +46,7 @@ boards: add_metric('CountBoardsMetric', time_frame: 'all'), There are several types of counters for metrics: -- **[Batch counters](#batch-counters)**: Used for counts and sums. +- **[Batch counters](#batch-counters)**: Used for counts, sums, and averages. - **[Redis counters](#redis-counters):** Used for in-memory counts. - **[Alternative counters](#alternative-counters):** Used for settings and configurations. @@ -102,34 +102,32 @@ Examples using `usage_data.rb` have been [deprecated](usage_data.md). We recomme #### Sum batch operation -There is no support for `sum` for database metrics. - Sum the values of a given ActiveRecord_Relation on given column and handles errors. Handles the `ActiveRecord::StatementInvalid` error Method: ```ruby -sum(relation, column, batch_size: nil, start: nil, finish: nil) +add_metric('JiraImportsTotalImportedIssuesCountMetric') ``` -Arguments: +#### Average batch operation -- `relation`: the ActiveRecord_Relation to perform the operation -- `column`: the column to sum on -- `batch_size`: if none set it uses default value 1000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting to avoid complex min calculations -- `end`: custom end of the batch counting to avoid complex min calculations +Average the values of a given `ActiveRecord_Relation` on given column and handles errors. -Examples: +Method: ```ruby -sum(JiraImportState.finished, :imported_issues_count) +add_metric('CountIssuesWeightAverageMetric') ``` +Examples: + +Examples using `usage_data.rb` have been [deprecated](usage_data.md). We recommend to use [instrumentation classes](metrics_instrumentation.md). + #### Grouping and batch operations -The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` +The `count`, `distinct_count`, `sum`, and `average` batch counters can accept an `ActiveRecord::Relation` object, which groups by a specified column. With a grouped relation, the methods do batch counting, handle errors, and returns a hash table of key-value pairs. @@ -144,6 +142,9 @@ distinct_count(Project.group(:visibility_level), :creator_id) sum(Issue.group(:state_id), :weight)) # returns => {1=>3542, 2=>6820} + +average(Issue.group(:state_id), :weight)) +# returns => {1=>3.5, 2=>2.5} ``` #### Add operation @@ -286,7 +287,7 @@ Enabled by default in GitLab 13.7 and later. Increment event count using an ordinary Redis counter, for a given event name. API requests are protected by checking for a valid CSRF token. - + ```plaintext POST /usage_data/increment_counter ``` @@ -652,9 +653,10 @@ We return fallback values in these cases: | Case | Value | |-----------------------------|-------| -| Deprecated Metric | -1000 | +| Deprecated Metric ([Removed with version 14.3](https://gitlab.com/gitlab-org/gitlab/-/issues/335894)) | -1000 | | Timeouts, general failures | -1 | | Standard errors in counters | -2 | +| Histogram metrics failure | { '-1' => -1 } | ## Test counters manually using your Rails console diff --git a/doc/development/service_ping/index.md b/doc/development/service_ping/index.md index 1e09dada36e..e776b78b710 100644 --- a/doc/development/service_ping/index.md +++ b/doc/development/service_ping/index.md @@ -464,7 +464,7 @@ To generate Service Ping, use [Teleport](https://goteleport.com/docs/) or a deta 1. Get the metrics duration from logs: -Search in Google Console logs for `time_elapsed`. Query example [here](https://cloudlogging.app.goo.gl/nWheZvD8D3nWazNe6). +Search in Google Console logs for `time_elapsed`. [Query example](https://cloudlogging.app.goo.gl/nWheZvD8D3nWazNe6). ### Verification (After approx 30 hours) diff --git a/doc/development/service_ping/metrics_dictionary.md b/doc/development/service_ping/metrics_dictionary.md index ead11a412fa..fee3bb571c2 100644 --- a/doc/development/service_ping/metrics_dictionary.md +++ b/doc/development/service_ping/metrics_dictionary.md @@ -35,7 +35,7 @@ Each metric is defined in a separate YAML file consisting of a number of fields: | `name` | no | Metric name suggestion. Can replace the last part of `key_path`. | | `description` | yes | | | `product_section` | yes | The [section](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/data/sections.yml). | -| `product_stage` | no | The [stage](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) for the metric. | +| `product_stage` | yes | The [stage](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) for the metric. | | `product_group` | yes | The [group](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) that owns the metric. | | `product_category` | no | The [product category](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/categories.yml) for the metric. | | `value_type` | yes | `string`; one of [`string`, `number`, `boolean`, `object`](https://json-schema.org/understanding-json-schema/reference/type.html). | @@ -43,11 +43,11 @@ Each metric is defined in a separate YAML file consisting of a number of fields: | `time_frame` | yes | `string`; may be set to a value like `7d`, `28d`, `all`, `none`. | | `data_source` | yes | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `system`. | | `data_category` | yes | `string`; [categories](#data-category) of the metric, may be set to `operational`, `optional`, `subscription`, `standard`. The default value is `optional`.| -| `instrumentation_class` | no | `string`; [the class that implements the metric](metrics_instrumentation.md). | +| `instrumentation_class` | yes | `string`; [the class that implements the metric](metrics_instrumentation.md). | | `distribution` | yes | `array`; may be set to one of `ce, ee` or `ee`. The [distribution](https://about.gitlab.com/handbook/marketing/strategic-marketing/tiers/#definitions) where the tracked feature is available. | | `performance_indicator_type` | no | `array`; may be set to one of [`gmau`, `smau`, `paid_gmau`, or `umau`](https://about.gitlab.com/handbook/business-technology/data-team/data-catalog/xmau-analysis/). | | `tier` | yes | `array`; may contain one or a combination of `free`, `premium` or `ultimate`. The [tier]( https://about.gitlab.com/handbook/marketing/strategic-marketing/tiers/) where the tracked feature is available. This should be verbose and contain all tiers where a metric is available. | -| `milestone` | no | The milestone when the metric is introduced and when it's available to self-managed instances with the official GitLab release. | +| `milestone` | yes | The milestone when the metric is introduced and when it's available to self-managed instances with the official GitLab release. | | `milestone_removed` | no | The milestone when the metric is removed. | | `introduced_by_url` | no | The URL to the merge request that introduced the metric to be available for self-managed instances. | | `removed_by_url` | no | The URL to the merge request that removed the metric. | diff --git a/doc/development/service_ping/metrics_instrumentation.md b/doc/development/service_ping/metrics_instrumentation.md index e718d972fba..4fd03eea84f 100644 --- a/doc/development/service_ping/metrics_instrumentation.md +++ b/doc/development/service_ping/metrics_instrumentation.md @@ -8,10 +8,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w This guide describes how to develop Service Ping metrics using metrics instrumentation. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a video tutorial, see the [Adding Service Ping metric via instrumentation class](https://youtu.be/p2ivXhNxUoY). + ## Nomenclature - **Instrumentation class**: - - Inherits one of the metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric` or `GenericMetric`. + - Inherits one of the metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric`, `NumbersMetric` or `GenericMetric`. - Implements the logic that calculates the value for a Service Ping metric. - **Metric definition** @@ -24,7 +27,7 @@ This guide describes how to develop Service Ping metrics using metrics instrumen A metric definition has the [`instrumentation_class`](metrics_dictionary.md) field, which can be set to a class. -The defined instrumentation class should inherit one of the existing metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric`, or `GenericMetric`. +The defined instrumentation class should inherit one of the existing metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric`, `NumbersMetric` or `GenericMetric`. The current convention is that a single instrumentation class corresponds to a single metric. On a rare occasions, there are exceptions to that convention like [Redis metrics](#redis-metrics). To use a single instrumentation class for more than one metric, please reach out to one of the `@gitlab-org/growth/product-intelligence/engineers` members to consult about your case. @@ -35,7 +38,7 @@ We have built a domain-specific language (DSL) to define the metrics instrumenta ## Database metrics -- `operation`: Operations for the given `relation`, one of `count`, `distinct_count`. +- `operation`: Operations for the given `relation`, one of `count`, `distinct_count`, `sum`, and `average`. - `relation`: `ActiveRecord::Relation` for the objects we want to perform the `operation`. - `start`: Specifies the start value of the batch counting, by default is `relation.minimum(:id)`. - `finish`: Specifies the end value of the batch counting, by default is `relation.maximum(:id)`. @@ -104,6 +107,46 @@ module Gitlab end ``` +### Sum Example + +```ruby +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraImportsTotalImportedIssuesCountMetric < DatabaseMetric + operation :sum, column: :imported_issues_count + + relation { JiraImportState.finished } + end + end + end + end +end +``` + +### Average Example + +```ruby +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountIssuesWeightAverageMetric < DatabaseMetric + operation :average, column: :weight + + relation { Issue } + end + end + end + end +end +``` + ## Redis metrics [Example of a merge request that adds a `Redis` metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66582). @@ -201,6 +244,43 @@ options: - i_quickactions_approve ``` +## Numbers metrics + +- `operation`: Operations for the given `data` block. Currently we only support `add` operation. +- `data`: a `block` which contains an array of numbers. +- `available?`: Specifies whether the metric should be reported. The default is `true`. + +```ruby +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IssuesBoardsCountMetric < NumbersMetric + operation :add + + data do |time_frame| + [ + CountIssuesMetric.new(time_frame: time_frame).value, + CountBoardsMetric.new(time_frame: time_frame).value + ] + end + end + end + end + end + end +end +``` + +You must also include the instrumentation class name in the YAML setup. + +```yaml +time_frame: 28d +instrumentation_class: 'IssuesBoardsCountMetric' +``` + ## Generic metrics - `value`: Specifies the value of the metric. @@ -228,14 +308,15 @@ end There is support for: -- `count`, `distinct_count`, `estimate_batch_distinct_count` for [database metrics](#database-metrics). +- `count`, `distinct_count`, `estimate_batch_distinct_count`, `sum`, and `average` for [database metrics](#database-metrics). - [Redis metrics](#redis-metrics). - [Redis HLL metrics](#redis-hyperloglog-metrics). +- `add` for [numbers metrics](#numbers-metrics). - [Generic metrics](#generic-metrics), which are metrics based on settings or configurations. There is no support for: -- `add`, `sum`, `histogram` for database metrics. +- `add`, `histogram` for database metrics. You can [track the progress to support these](https://gitlab.com/groups/gitlab-org/-/epics/6118). @@ -245,8 +326,10 @@ To create a stub instrumentation for a Service Ping metric, you can use a dedica The generator takes the class name as an argument and the following options: -- `--type=TYPE` Required. Indicates the metric type. It must be one of: `database`, `generic`, `redis`. -- `--operation` Required for `database` type. It must be one of: `count`, `distinct_count`, `estimate_batch_distinct_count`. +- `--type=TYPE` Required. Indicates the metric type. It must be one of: `database`, `generic`, `redis`, `numbers`. +- `--operation` Required for `database` & `numebers` type. + - For `database` it must be one of: `count`, `distinct_count`, `estimate_batch_distinct_count`, `sum`, `average`. + - For `numbers` it must be: `add`. - `--ee` Indicates if the metric is for EE. ```shell @@ -264,6 +347,7 @@ This guide describes how to migrate a Service Ping metric from [`lib/gitlab/usag - [Database metric](#database-metrics) - [Redis HyperLogLog metrics](#redis-hyperloglog-metrics) - [Redis metric](#redis-metrics) +- [Numbers metric](#numbers-metrics) - [Generic metric](#generic-metrics) 1. Determine the location of instrumentation class: either under `ee` or outside `ee`. diff --git a/doc/development/shell_commands.md b/doc/development/shell_commands.md index 6f56e60f619..fcb8c20bdd3 100644 --- a/doc/development/shell_commands.md +++ b/doc/development/shell_commands.md @@ -17,7 +17,7 @@ These guidelines are meant to make your code more reliable _and_ secure. ## Use File and FileUtils instead of shell commands -Sometimes we invoke basic Unix commands via the shell when there is also a Ruby API for doing it. Use the Ruby API if it exists. <http://www.ruby-doc.org/stdlib-2.0.0/libdoc/fileutils/rdoc/FileUtils.html#module-FileUtils-label-Module+Functions> +Sometimes we invoke basic Unix commands via the shell when there is also a Ruby API for doing it. Use the Ruby API if it exists. <https://www.ruby-doc.org/stdlib-2.0.0/libdoc/fileutils/rdoc/FileUtils.html#module-FileUtils-label-Module+Functions> ```ruby # Wrong diff --git a/doc/development/sidekiq/logging.md b/doc/development/sidekiq/logging.md index 015376b0fc6..474ea5de951 100644 --- a/doc/development/sidekiq/logging.md +++ b/doc/development/sidekiq/logging.md @@ -25,7 +25,7 @@ need to do anything. There are however some instances when there would be no context present when the job is scheduled, or the context that is present is -likely to be incorrect. For these instances, we've added Rubocop rules +likely to be incorrect. For these instances, we've added RuboCop rules to draw attention and avoid incorrect metadata in our logs. As with most our cops, there are perfectly valid reasons for disabling diff --git a/doc/development/sidekiq/worker_attributes.md b/doc/development/sidekiq/worker_attributes.md index 3bd6d313e2c..6820627f761 100644 --- a/doc/development/sidekiq/worker_attributes.md +++ b/doc/development/sidekiq/worker_attributes.md @@ -6,6 +6,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Sidekiq worker attributes +Worker classes can define certain attributes to control their behavior and add metadata. + +Child classes inheriting from other workers also inherit these attributes, so you only +have to redefine them if you want to override their values. + ## Job urgency Jobs can have an `urgency` attribute set, which can be `:high`, diff --git a/doc/development/single_table_inheritance.md b/doc/development/single_table_inheritance.md index 0783721e628..c8d082e8a67 100644 --- a/doc/development/single_table_inheritance.md +++ b/doc/development/single_table_inheritance.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/snowplow/implementation.md b/doc/development/snowplow/implementation.md index f4123e3ba86..88fb1d5cfe4 100644 --- a/doc/development/snowplow/implementation.md +++ b/doc/development/snowplow/implementation.md @@ -464,7 +464,10 @@ Page titles are hardcoded as `GitLab` for the same reason. #### Snowplow Inspector Chrome Extension -Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works in production, staging, and local development environments. +Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works in production, staging, and local development environments. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a video tutorial, see the [Snowplow plugin walk through](https://www.youtube.com/watch?v=g4rqnIZ1Mb4). 1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). 1. To open the extension, select the Snowplow Inspector icon beside the address bar. diff --git a/doc/development/snowplow/index.md b/doc/development/snowplow/index.md index 9b684757fe1..d6a7b900629 100644 --- a/doc/development/snowplow/index.md +++ b/doc/development/snowplow/index.md @@ -78,6 +78,8 @@ sequenceDiagram Snowflake DW->>Sisense Dashboards: Data available for querying ``` +For more details about the architecture, see [Snowplow infrastructure](infrastructure.md). + ## Structured event taxonomy Click events must be consistent. If each feature captures events differently, it can be difficult @@ -184,19 +186,6 @@ LIMIT 100 Snowplow JavaScript adds [web-specific parameters](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/snowplow-tracker-protocol/#Web-specific_parameters) to all web events by default. -## Snowplow monitoring - -For different stages in the processing pipeline, there are several tools that monitor Snowplow events tracking: - -- [Product Intelligence Grafana dashboard](https://dashboards.gitlab.net/d/product-intelligence-main/product-intelligence-product-intelligence?orgId=1) monitors backend events sent from GitLab.com instance to collectors fleet. This dashboard provides information about: - - The number of events that successfully reach Snowplow collectors. - - The number of events that failed to reach Snowplow collectors. - - The number of backend events that were sent. -- [AWS CloudWatch dashboard](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards:name=SnowPlow;start=P3D) monitors the state of the events processing pipeline. The pipeline starts from Snowplow collectors, through to enrichers and pseudonymization, and up to persistence on S3 bucket from which events are imported to Snowflake Data Warehouse. To view this dashboard AWS access is required, follow this [instruction](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization#monitoring) if you are interested in getting one. -- [SiSense dashboard](https://app.periscopedata.com/app/gitlab/417669/Snowplow-Summary-Dashboard) provides information about the number of good and bad events imported into the Data Warehouse, in addition to the total number of imported Snowplow events. - -For more information, see this [video walk-through](https://www.youtube.com/watch?v=NxPS0aKa_oU). - ## Related topics - [Snowplow data structure](https://docs.snowplowanalytics.com/docs/understanding-your-pipeline/canonical-event/) diff --git a/doc/development/snowplow/infrastructure.md b/doc/development/snowplow/infrastructure.md new file mode 100644 index 00000000000..28541874e98 --- /dev/null +++ b/doc/development/snowplow/infrastructure.md @@ -0,0 +1,101 @@ +--- +stage: Growth +group: Product Intelligence +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Snowplow infrastructure + +Snowplow events on GitLab SaaS fired by a [tracker](implementation.md) go through an AWS pipeline, managed by GitLab. + +## Event flow in the AWS pipeline + +Every event goes through a collector, enricher, and pseudonymization lambda. The event is then dumped to S3 storage where it can be picked up by the Snowflake data warehouse. + +Deploying and managing the infrastructure is automated using Terraform in the current [Terraform repository](https://gitlab.com/gitlab-com/gl-infra/config-mgmt/-/tree/master/environments/aws-snowplow). + +```mermaid +graph LR + GL[GitLab.com]-->COL + + subgraph aws-cloud[AWS] + COL[Collector]-->|snowplow-raw-good|ENR + COL[Collector]-->|snowplow-raw-bad|FRBE + subgraph firehoserbe[Firehose] + FRBE[AWS Lambda] + end + FRBE-->S3RBE + + ENR[Enricher]-->|snowplow-enriched-bad|FEBE + subgraph firehoseebe[Firehose] + FEBE[AWS Lambda] + end + FEBE-->S3EBE + + ENR[Enricher]-->|snowplow-enriched-good|FRGE + subgraph firehosege[Firehose] + FRGE[AWS Lambda] + end + FRGE-->S3GE + end + + subgraph snowflake[Data warehouse] + S3RBE[S3 raw-bad]-->BE[gitlab_bad_events] + S3EBE[S3 enriched-bad]-->BE[gitlab_bad_events] + S3GE[S3 output]-->GE[gitlab_events] + end +``` + +See [Snowplow technology 101](https://github.com/snowplow/snowplow/#snowplow-technology-101) for Snowplow's own documentation and an overview how collectors and enrichers work. + +### Pseudonymization + +In contrast to a typical Snowplow pipeline, after enrichment, GitLab Snowplow events go through a [pseudonymization service](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization) in the form of an AWS Lambda service before they are stored in S3 storage. + +#### Why events need to be pseudonymized + +GitLab is bound by its [obligations to community](https://about.gitlab.com/handbook/product/product-intelligence-guide/service-usage-data-commitment/) +and by [legal regulations](https://about.gitlab.com/handbook/legal/privacy/services-usage-data/) to protect the privacy of its users. + +GitLab must provide valuable insights for business decisions, and there is a need +for a better understanding of different users' behavior patterns. The +pseudonymization process helps you find a compromise between these two requirements. + +Pseudonymization processes personally identifiable information inside a Snowplow event in an irreversible fashion +maintaining deterministic output for given input, while masking any relation to that input. + +#### How events are pseudonymized + +Pseudonymization uses an allowlist that provides privacy by default. Therefore, each +attribute received as part of a Snowplow event is pseudonymized unless the attribute +is an allowed exception. + +Pseudonymization is done using the HMAC-SHA256 keyed hash algorithm. +Attributes are combined with a secret salt to replace each identifiable information with a pseudonym. + +### S3 bucket data lake to Snowflake + +See [Data team's Snowplow Overview](https://about.gitlab.com/handbook/business-technology/data-team/platform/snowplow/) for further details how data is ingested into our Snowflake data warehouse. + +## Monitoring + +There are several tools that monitor Snowplow events tracking in different stages of the processing pipeline: + +- [Product Intelligence Grafana dashboard](https://dashboards.gitlab.net/d/product-intelligence-main/product-intelligence-product-intelligence?orgId=1) monitors backend events sent from a GitLab.com instance to a collectors fleet. This dashboard provides information about: + - The number of events that successfully reach Snowplow collectors. + - The number of events that failed to reach Snowplow collectors. + - The number of backend events that were sent. +- [AWS CloudWatch dashboard](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards:name=SnowPlow;start=P3D) monitors the state of the events in a processing pipeline. The pipeline starts from Snowplow collectors, goes through to enrichers and pseudonymization, and then up to persistence in an S3 bucket. From S3, the events are imported into the Snowflake Data Warehouse. You must have AWS access rights to view this dashboard. For more information, see [monitoring](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization#monitoring) in the Snowplow Events pseudonymization service documentation. +- [Sisense dashboard](https://app.periscopedata.com/app/gitlab/417669/Snowplow-Summary-Dashboard) provides information about the number of good and bad events imported into the Data Warehouse, in addition to the total number of imported Snowplow events. + +For more information, see this [video walk-through](https://www.youtube.com/watch?v=NxPS0aKa_oU). + +## Related topics + +- [Snowplow technology 101](https://github.com/snowplow/snowplow/#snowplow-technology-101) +- [Snowplow pseudonymization AWS Lambda project](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization) +- [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-technology/data-team/platform/infrastructure/) +- [Snowplow architecture overview (internal)](https://www.youtube.com/watch?v=eVYJjzspsLU) +- [Snowplow architecture overview slide deck (internal)](https://docs.google.com/presentation/d/16gQEO5CAg8Tx4NBtfnZj-GF4juFI6HfEPWcZgH4Rn14/edit?usp=sharing) +- [AWS Lambda implementation (internal)](https://youtu.be/cQd0mdMhkQA) diff --git a/doc/development/sql.md b/doc/development/sql.md index 4b6153b7205..8553e2a5500 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w This document describes various guidelines to follow when writing SQL queries, either using ActiveRecord/Arel or raw SQL queries. -## Using LIKE Statements +## Using `LIKE` Statements The most common way to search for data is using the `LIKE` statement. For example, to get all issues with a title starting with "Draft:" you'd write the @@ -56,10 +56,10 @@ FROM issues WHERE (title ILIKE 'Draft:%' OR foo ILIKE 'Draft:%') ``` -## LIKE & Indexes +## `LIKE` & Indexes -PostgreSQL won't use any indexes when using `LIKE` / `ILIKE` with a wildcard at -the start. For example, this will not use any indexes: +PostgreSQL does not use any indexes when using `LIKE` / `ILIKE` with a wildcard at +the start. For example, this does not use any indexes: ```sql SELECT * @@ -145,7 +145,7 @@ The query: Project.select("path, user_id").joins(:merge_requests) # SELECT path, user_id FROM "projects" ... ``` -Later on, a new feature adds an extra column to the `projects` table: `user_id`. During deployment there might be a short time window where the database migration is already executed, but the new version of the application code is not deployed yet. When the query mentioned above executes during this period, the query will fail with the following error message: `PG::AmbiguousColumn: ERROR: column reference "user_id" is ambiguous` +Later on, a new feature adds an extra column to the `projects` table: `user_id`. During deployment there might be a short time window where the database migration is already executed, but the new version of the application code is not deployed yet. When the query mentioned above executes during this period, the query fails with the following error message: `PG::AmbiguousColumn: ERROR: column reference "user_id" is ambiguous` The problem is caused by the way the attributes are selected from the database. The `user_id` column is present in both the `users` and `merge_requests` tables. The query planner cannot decide which table to use when looking up the `user_id` column. @@ -210,7 +210,7 @@ Project.select(:path, :user_id).joins(:merge_requests) # SELECT "projects"."path", "user_id" FROM "projects" ... ``` -When a column list is given, ActiveRecord tries to match the arguments against the columns defined in the `projects` table and prepend the table name automatically. In this case, the `id` column is not going to be a problem, but the `user_id` column could return unexpected data: +When a column list is given, ActiveRecord tries to match the arguments against the columns defined in the `projects` table and prepend the table name automatically. In this case, the `id` column is not a problem, but the `user_id` column could return unexpected data: ```ruby Project.select(:id, :user_id).joins(:merge_requests) @@ -225,7 +225,7 @@ Project.select(:id, :user_id).joins(:merge_requests) ## Plucking IDs Never use ActiveRecord's `pluck` to pluck a set of values into memory only to -use them as an argument for another query. For example, this will execute an +use them as an argument for another query. For example, this executes an extra unnecessary database query and load a lot of unnecessary data into memory: ```ruby @@ -314,10 +314,10 @@ union = Gitlab::SQL::Union.new([projects, more_projects, ...]) Project.from("(#{union.to_sql}) projects") ``` -### Uneven columns in the UNION sub-queries +### Uneven columns in the `UNION` sub-queries -When the UNION query has uneven columns in the SELECT clauses, the database returns an error. -Consider the following UNION query: +When the `UNION` query has uneven columns in the `SELECT` clauses, the database returns an error. +Consider the following `UNION` query: ```sql SELECT id FROM users WHERE id = 1 @@ -333,7 +333,7 @@ each UNION query must have the same number of columns ``` This problem is apparent and it can be easily fixed during development. One edge-case is when -UNION queries are combined with explicit column listing where the list comes from the +`UNION` queries are combined with explicit column listing where the list comes from the `ActiveRecord` schema cache. Example (bad, avoid it): @@ -387,17 +387,17 @@ User.connection.execute(Gitlab::SQL::Union.new([scope1, scope2]).to_sql) When ordering records based on the time they were created, you can order by the `id` column instead of ordering by `created_at`. Because IDs are always -unique and incremented in the order that rows are created, doing so will produce the +unique and incremented in the order that rows are created, doing so produces the exact same results. This also means there's no need to add an index on `created_at` to ensure consistent performance as `id` is already indexed by default. -## Use WHERE EXISTS instead of WHERE IN +## Use `WHERE EXISTS` instead of `WHERE IN` While `WHERE IN` and `WHERE EXISTS` can be used to produce the same data it is recommended to use `WHERE EXISTS` whenever possible. While in many cases PostgreSQL can optimise `WHERE IN` quite well there are also many cases where -`WHERE EXISTS` will perform (much) better. +`WHERE EXISTS` performs (much) better. In Rails you have to use this by creating SQL fragments: @@ -446,7 +446,7 @@ method. This method differs from our `.safe_find_or_create_by` methods because it performs the `INSERT`, and then performs the `SELECT` commands only if that call fails. -If the `INSERT` fails, it will leave a dead tuple around and +If the `INSERT` fails, it leaves a dead tuple around and increment the primary key sequence (if any), among [other downsides](https://api.rubyonrails.org/classes/ActiveRecord/Relation.html#method-i-create_or_find_by). We prefer `.safe_find_or_create_by` if the common path is that we diff --git a/doc/development/swapping_tables.md b/doc/development/swapping_tables.md index cb038a3b85a..efb481ccf35 100644 --- a/doc/development/swapping_tables.md +++ b/doc/development/swapping_tables.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -10,12 +10,12 @@ Sometimes you need to replace one table with another. For example, when migrating data in a very large table it's often better to create a copy of the table and insert & migrate the data into this new table in the background. -Let's say you want to swap the table "events" with "events_for_migration". In +Let's say you want to swap the table `events` with `events_for_migration`. In this case you need to follow 3 steps: -1. Rename "events" to "events_temporary" -1. Rename "events_for_migration" to "events" -1. Rename "events_temporary" to "events_for_migration" +1. Rename `events` to `events_temporary` +1. Rename `events_for_migration` to `events` +1. Rename `events_temporary` to `events_for_migration` Rails allows you to do this using the `rename_table` method: @@ -27,7 +27,7 @@ rename_table :events_temporary, :events_for_migration This does not require any downtime as long as the 3 `rename_table` calls are executed in the _same_ database transaction. Rails by default uses database -transactions for migrations, but if it doesn't you'll need to start one +transactions for migrations, but if it doesn't you need to start one manually: ```ruby @@ -45,7 +45,7 @@ PostgreSQL you can use the `reset_pk_sequence!` method like so: reset_pk_sequence!('events') ``` -Failure to reset the primary keys will result in newly created rows starting +Failure to reset the primary keys results in newly created rows starting with an ID value of 1. Depending on the existing data this can then lead to duplicate key constraints from popping up, preventing users from creating new data. diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 7ae49d33e91..eda1c8c3d10 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -215,6 +215,39 @@ In this case, the `total time` and `top-level time` numbers match more closely: 8 8 0.0477s 0.0477s 0.0477s namespace ``` +#### Stubbing methods within factories + +You should avoid using `allow(object).to receive(:method)` in factories, as this makes the factory unable to be used with `let_it_be`. + +Instead, you can use `stub_method` to stub the method: + +```ruby + before(:create) do |user, evaluator| + # Stub a method. + stub_method(user, :some_method) { 'stubbed!' } + # Or with arguments, including named ones + stub_method(user, :some_method) { |var1| "Returning #{var1}!" } + stub_method(user, :some_method) { |var1: 'default'| "Returning #{var1}!" } + end + + # Un-stub the method. + # This may be useful where the stubbed object is created with `let_it_be` + # and you want to reset the method between tests. + after(:create) do |user, evaluator| + restore_original_method(user, :some_method) + # or + restore_original_methods(user) + end +``` + +NOTE: +`stub_method` does not work when used in conjunction with `let_it_be_with_refind`. This is because `stub_method` will stub a method on an instance and `let_it_be_with_refind` will create a new instance of the object for each run. + +`stub_method` does not support method existence and method arity checks. + +WARNING: +`stub_method` is supposed to be used in factories only. It's strongly discouraged to be used elsewhere. Please consider using [RSpec's mocks](https://relishapp.com/rspec/rspec-mocks/v/3-10/docs/basics) if available. + #### Identify slow tests Running a spec with profiling is a good way to start optimizing a spec. This can @@ -981,7 +1014,7 @@ is used to delete data in all indices in between examples to ensure a clean inde Note that Elasticsearch indexing uses [`Gitlab::Redis::SharedState`](../../../ee/development/redis.md#gitlabrediscachesharedstatequeues). Therefore, the Elasticsearch traits dynamically use the `:clean_gitlab_redis_shared_state` trait. -You do NOT need to add `:clean_gitlab_redis_shared_state` manually. +You do not need to add `:clean_gitlab_redis_shared_state` manually. Specs using Elasticsearch require that you: @@ -1305,7 +1338,7 @@ GitLab uses [factory_bot](https://github.com/thoughtbot/factory_bot) as a test f See [issue #262624](https://gitlab.com/gitlab-org/gitlab/-/issues/262624) for further context. - Factories don't have to be limited to `ActiveRecord` objects. [See example](https://gitlab.com/gitlab-org/gitlab-foss/commit/0b8cefd3b2385a21cfed779bd659978c0402766d). -- Factories and their traits should produce valid objects that are [verified by specs](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/factories_spec.rb). +- Factories and their traits should produce valid objects that are [verified by specs](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/models/factories_spec.rb). - Avoid the use of [`skip_callback`](https://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-skip_callback) in factories. See [issue #247865](https://gitlab.com/gitlab-org/gitlab/-/issues/247865) for details. diff --git a/doc/development/testing_guide/contract/consumer_tests.md b/doc/development/testing_guide/contract/consumer_tests.md new file mode 100644 index 00000000000..b4d6882a655 --- /dev/null +++ b/doc/development/testing_guide/contract/consumer_tests.md @@ -0,0 +1,308 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Writing consumer tests + +This tutorial guides you through writing a consumer test from scratch. To start, the consumer tests are written using [`jest-pact`](https://github.com/pact-foundation/jest-pact) that builds on top of [`pact-js`](https://github.com/pact-foundation/pact-js). This tutorial shows you how to write a consumer test for the `/discussions.json` endpoint, which is actually `/:namespace_name/:project_name/-/merge_requests/:id/discussions.json`. + +## Create the skeleton + +Start by creating the skeleton of a consumer test. Create a file under `spec/contracts/consumer/specs` called `discussions.spec.js`. +Then, populate it with the following function and parameters: + +- [`pactWith`](#the-pactwith-function) +- [`PactOptions`](#the-pactoptions-parameter) +- [`PactFn`](#the-pactfn-parameter) + +### The `pactWith` function + +The Pact consumer test is defined through the `pactWith` function that takes `PactOptions` and the `PactFn`. + +```javascript +const { pactWith } = require('jest-pact'); + +pactWith(PactOptions, PactFn); +``` + +### The `PactOptions` parameter + +`PactOptions` with `jest-pact` introduces [additional options](https://github.com/pact-foundation/jest-pact/blob/dce370c1ab4b7cb5dff12c4b62246dc229c53d0e/README.md#defaults) that build on top of the ones [provided in `pact-js`](https://github.com/pact-foundation/pact-js#constructor). In most cases, you define the `consumer`, `provider`, `log`, and `dir` options for these tests. + +```javascript +const { pactWith } = require('jest-pact'); + +pactWith( + { + consumer: 'Merge Request Page', + provider: 'Merge Request Discussions Endpoint', + log: '../logs/consumer.log', + dir: '../contracts', + }, + PactFn +); +``` + +### The `PactFn` parameter + +The `PactFn` is where your tests are defined. This is where you set up the mock provider and where you can use the standard Jest methods like [`Jest.describe`](https://jestjs.io/docs/api#describename-fn), [`Jest.beforeEach`](https://jestjs.io/docs/api#beforeeachfn-timeout), and [`Jest.it`](https://jestjs.io/docs/api#testname-fn-timeout). For more information, see [https://jestjs.io/docs/api](https://jestjs.io/docs/api). + +```javascript +const { pactWith } = require('jest-pact'); + +pactWith( + { + consumer: 'Merge Request Page', + provider: 'Merge Request Discussions Endpoint', + log: '../logs/consumer.log', + dir: '../contracts', + }, + + (provider) => { + describe('Discussions Endpoint', () => { + beforeEach(() => { + + }); + + it('return a successful body', () => { + + }); + }); + }, +); +``` + +## Set up the mock provider + +Before you run your test, set up the mock provider that handles the specified requests and returns a specified response. To do that, define the state and the expected request and response in an [`Interaction`](https://github.com/pact-foundation/pact-js/blob/master/src/dsl/interaction.ts). + +For this tutorial, define four attributes for the `Interaction`: + +1. `state`: A description of what the prerequisite state is before the request is made. +1. `uponReceiving`: A description of what kind of request this `Interaction` is handling. +1. `withRequest`: Where you define the request specifications. It contains the request `method`, `path`, and any `headers`, `body`, or `query`. +1. `willRespondWith`: Where you define the expected response. It contains the response `status`, `headers`, and `body`. + +After you define the `Interaction`, add that interaction to the mock provider by calling `addInteraction`. + +```javascript +const { pactWith } = require('jest-pact'); +const { Matchers } = require('@pact-foundation/pact'); + +pactWith( + { + consumer: 'Merge Request Page', + provider: 'Merge Request Discussions Endpoint', + log: '../logs/consumer.log', + dir: '../contracts', + }, + + (provider) => { + describe('Discussions Endpoint', () => { + beforeEach(() => { + const interaction = { + state: 'a merge request with discussions exists', + uponReceiving: 'a request for discussions', + withRequest: { + method: 'GET', + path: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', + headers: { + Accept: '*/*', + }, + }, + willRespondWith: { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: Matchers.eachLike({ + id: Matchers.string('fd73763cbcbf7b29eb8765d969a38f7d735e222a'), + project_id: Matchers.integer(6954442), + ... + resolved: Matchers.boolean(true) + }), + }, + }; + provider.addInteraction(interaction); + }); + + it('return a successful body', () => { + + }); + }); + }, +); +``` + +### Response body `Matchers` + +Notice how we use `Matchers` in the `body` of the expected response. This allows us to be flexible enough to accept different values but still be strict enough to distinguish between valid and invalid values. We must ensure that we have a tight definition that is neither too strict nor too lax. Read more about the [different types of `Matchers`](https://github.com/pact-foundation/pact-js#using-the-v3-matching-rules). + +## Write the test + +After the mock provider is set up, you can write the test. For this test, you make a request and expect a particular response. + +First, set up the client that makes the API request. To do that, either create or find an existing file under `spec/contracts/consumer/endpoints` and add the following API request. + +```javascript +const axios = require('axios'); + +exports.getDiscussions = (endpoint) => { + const url = endpoint.url; + + return axios + .request({ + method: 'GET', + baseURL: url, + url: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', + headers: { Accept: '*/*' }, + }) + .then((response) => response.data); +}; +``` + +After that's set up, import it to the test file and call it to make the request. Then, you can make the request and define your expectations. + +```javascript +const { pactWith } = require('jest-pact'); +const { Matchers } = require('@pact-foundation/pact'); + +const { getDiscussions } = require('../endpoints/merge_requests'); + +pactWith( + { + consumer: 'Merge Request Page', + provider: 'Merge Request Discussions Endpoint', + log: '../logs/consumer.log', + dir: '../contracts', + }, + + (provider) => { + describe('Discussions Endpoint', () => { + beforeEach(() => { + const interaction = { + state: 'a merge request with discussions exists', + uponReceiving: 'a request for discussions', + withRequest: { + method: 'GET', + path: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', + headers: { + Accept: '*/*', + }, + }, + willRespondWith: { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: Matchers.eachLike({ + id: Matchers.string('fd73763cbcbf7b29eb8765d969a38f7d735e222a'), + project_id: Matchers.integer(6954442), + ... + resolved: Matchers.boolean(true) + }), + }, + }; + }); + + it('return a successful body', () => { + return getDiscussions({ + url: provider.mockService.baseUrl, + }).then((discussions) => { + expect(discussions).toEqual(Matchers.eachLike({ + id: 'fd73763cbcbf7b29eb8765d969a38f7d735e222a', + project_id: 6954442, + ... + resolved: true + })); + }); + }); + }); + }, +); +``` + +There we have it! The consumer test is now set up. You can now try [running this test](index.md#run-the-consumer-tests). + +## Improve test readability + +As you may have noticed, the request and response definitions can get large. This results in the test being difficult to read, with a lot of scrolling to find what you want. You can make the test easier to read by extracting these out to a `fixture`. + +Create a file under `spec/contracts/consumer/fixtures` called `discussions.fixture.js`. You place the `request` and `response` definitions here. + +```javascript +const { Matchers } = require('@pact-foundation/pact'); + +const body = Matchers.eachLike({ + id: Matchers.string('fd73763cbcbf7b29eb8765d969a38f7d735e222a'), + project_id: Matchers.integer(6954442), + ... + resolved: Matchers.boolean(true) +}); + +const Discussions = { + body: Matchers.extractPayload(body), + + success: { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + body: body, + }, + + request: { + uponReceiving: 'a request for discussions', + withRequest: { + method: 'GET', + path: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', + headers: { + Accept: '*/*', + }, + }, + }, +}; + +exports.Discussions = Discussions; +``` + +With all of that moved to the `fixture`, you can simplify the test to the following: + +```javascript +const { pactWith } = require('jest-pact'); + +const { Discussions } = require('../fixtures/discussions.fixture'); +const { getDiscussions } = require('../endpoints/merge_requests'); + +pactWith( + { + consumer: 'Merge Request Page', + provider: 'Merge Request Discussions Endpoint', + log: '../logs/consumer.log', + dir: '../contracts', + }, + + (provider) => { + describe('Discussions Endpoint', () => { + beforeEach(() => { + const interaction = { + state: 'a merge request with discussions exists', + ...Discussions.request, + willRespondWith: Discussions.success, + }; + return provider.addInteraction(interaction); + }); + + it('return a successful body', () => { + return getDiscussions({ + url: provider.mockService.baseUrl, + }).then((discussions) => { + expect(discussions).toEqual(Discussions.body); + }); + }); + }); + }, +); +``` diff --git a/doc/development/testing_guide/contract/index.md b/doc/development/testing_guide/contract/index.md new file mode 100644 index 00000000000..6556bd85624 --- /dev/null +++ b/doc/development/testing_guide/contract/index.md @@ -0,0 +1,39 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Contract testing + +Contract tests consist of two parts — consumer tests and provider tests. A simple example of a consumer and provider relationship is between the frontend and backend. The frontend would be the consumer and the backend is the provider. The frontend consumes the API that is provided by the backend. The test helps ensure that these two sides follow an agreed upon contract and any divergence from the contract triggers a meaningful conversation to prevent breaking changes from slipping through. + +Consumer tests are similar to unit tests with each spec defining a requests and an expected mock responses and creating a contract based on those definitions. On the other hand, provider tests are similar to integration tests as each spec takes the request defined in the contract and runs that request against the actual service which is then matched against the contract to validate the contract. + +You can check out the existing contract tests at: + +- [`spec/contracts/consumer/specs`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/consumer/specs) for the consumer tests. +- [`spec/contracts/provider/specs`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/specs) for the provider tests. + +The contracts themselves are stored in [`/spec/contracts/contracts`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/contracts) at the moment. The plan is to use [PactBroker](https://docs.pact.io/pact_broker/docker_images) hosted in AWS or another similar service. + +## Write the tests + +- [Writing consumer tests](consumer_tests.md) +- [Writing provider tests](provider_tests.md) + +### Run the consumer tests + +Before running the consumer tests, go to `spec/contracts/consumer` and run `npm install`. To run all the consumer tests, you just need to run `npm test -- /specs`. Otherwise, to run a specific spec file, replace `/specs` with the specific spec filename. + +### Run the provider tests + +Before running the provider tests, make sure your GDK (GitLab Development Kit) is fully set up and running. You can follow the setup instructions detailed in the [GDK repository](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main). To run the provider tests, you use Rake tasks that are defined in [`./lib/tasks/contracts.rake`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/tasks/contracts.rake). To get a list of all the Rake tasks related to the provider tests, run `bundle exec rake -T contracts`. For example: + +```shell +$ bundle exec rake -T contracts +rake contracts:mr:pact:verify:diffs # Verify provider against the consumer pacts for diffs +rake contracts:mr:pact:verify:discussions # Verify provider against the consumer pacts for discussions +rake contracts:mr:pact:verify:metadata # Verify provider against the consumer pacts for metadata +rake contracts:mr:test:merge_request[contract_mr] # Run all merge request contract tests +``` diff --git a/doc/development/testing_guide/contract/provider_tests.md b/doc/development/testing_guide/contract/provider_tests.md new file mode 100644 index 00000000000..0da5bcb4aef --- /dev/null +++ b/doc/development/testing_guide/contract/provider_tests.md @@ -0,0 +1,177 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Writing provider tests + +This tutorial guides you through writing a provider test from scratch. It is a continuation of the [consumer test tutorial](consumer_tests.md). To start, the provider tests are written using [`pact-ruby`](https://github.com/pact-foundation/pact-ruby). In this tutorial, you write a provider test that addresses the contract generated by `discussions.spec.js`. + +## Create the skeleton + +Provider tests are quite simple. The goal is to set up the test data and then link that with the corresponding contract. Start by creating a file called `discussions_helper.rb` under `spec/contracts/provider/specs`. Note that the files are called `helpers` to match how they are called by Pact in the Rake tasks, which are set up at the end of this tutorial. + +### The `service_provider` block + +The `service_provider` block is where the provider test is defined. For this block, put in a description of the service provider. Name it exactly as it is called in the contracts that are derived from the consumer tests. + +```ruby +require_relative '../spec_helper' + +module Provider + module DiscussionsHelper + Pact.service_provider 'Merge Request Discussions Endpoint' do + + end + end +end +``` + +### The `honours_pact_with` block + +The `honours_pact_with` block describes which consumer this provider test is addressing. Similar to the `service_provider` block, name this exactly the same as it's called in the contracts that are derived from the consumer tests. + +```ruby +require_relative '../spec_helper' + +module Provider + module DiscussionsHelper + Pact.service_provider 'Merge Request Discussions Endpoint' do + honours_pact_with 'Merge Request Page' do + + end + end + end +end +``` + +## Configure the test app + +For the provider tests to verify the contracts, you must hook it up to a test app that makes the actual request and return a response to verify against the contract. To do this, configure the `app` the test uses as `Environment::Test.app`, which is defined in [`spec/contracts/provider/environments/test.rb`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/environments/test.rb). + +```ruby +require_relative '../spec_helper' + +module Provider + module DiscussionsHelper + Pact.service_provider 'Merge Request Discussions Endpoint' do + app { Environment::Test.app } + + honours_pact_with 'Merge Request Page' do + + end + end + end +end +``` + +## Define the contract to verify + +Now that the test app is configured, all that is left is to define which contract this provider test is verifying. To do this, set the `pact_uri`. + +```ruby +require_relative '../spec_helper' + +module Provider + module DiscussionsHelper + Pact.service_provider 'Merge Request Discussions Endpoint' do + app { Environment::Test.app } + + honours_pact_with 'Merge Request Page' do + pact_uri '../contracts/merge_request_page-merge_request_discussions_endpoint.json' + end + end + end +end +``` + +## Add / update the Rake tasks + +Now that you have a test created, you must create Rake tasks that run this test. The Rake tasks are defined in [`lib/tasks/contracts.rake`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/tasks/contracts.rake) where we have individual Rake tasks to run individual specs, but also Rake tasks that run a group of tests. + +Under the `contracts:mr` namespace, introduce the Rake task to run this new test specifically. In it, call `pact.uri` to define the location of the contract and the provider test that tests that contract. Notice here that `pact_uri` has a parameter called `pact_helper`. This is why the provider tests are called `_helper.rb`. + +```ruby +Pact::VerificationTask.new(:discussions) do |pact| + pact.uri( + "#{contracts}/contracts/merge_request_page-merge_request_discussions_endpoint.json", + pact_helper: "#{provider}/specs/discussions_helper.rb" + ) +end +``` + +At the same time, add your new `:discussions` Rake task to be included in the `test:merge_request` Rake task. In that Rake task, there is an array defined (`%w[metadata diffs]`). You must add `discussions` in that list. + +## Create test data + +As the last step, create the test data that allows the provider test to return the contract's expected response. You might wonder why you create the test data last. It's really a matter of preference. With the test already configured, you can easily run the test to verify and make sure all the necessary test data are created to produce the expected response. + +You can read more about [provider states](https://docs.pact.io/implementation_guides/ruby/provider_states). We can do global provider states but for this tutorial, the provider state is for one specific `state`. + +To create the test data, create `discussions_state.rb` under `spec/contracts/provider/states`. As a quick aside, make sure to also import this state file in the `discussions_helper.rb` file. + +### Default user in `spec/contracts/provider/spec_helper.rb` + +Before you create the test data, note that a default user is created in the [`spec_helper`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/spec_helper.rb), which is the user being used for the test runs. This user is configured using `RSpec.configure`, as Pact actually is built on top of RSpec. This step allows us to configure the user before any of the test runs. + +```ruby +RSpec.configure do |config| + config.include Devise::Test::IntegrationHelpers + config.before do + user = FactoryBot.create(:user, name: "Contract Test").tap do |user| + user.current_sign_in_at = Time.current + end + sign_in user + end +end +``` + +Any further modifications to the user that's needed can be done through the individual provider state files. + +### The `provider_states_for` block + +In the state file, you must define which consumer this provider state is for. You can do that with `provider_states_for`. Make sure that the `name` provided matches the name defined for the consumer. + +```ruby +Pact.provider_states_for 'Merge Request Page' do +end +``` + +### The `provider_state` block + +In the `provider_states_for` block, you then define the state the test data is for. These states are also defined in the consumer test. In this case, there is a `'a merge request with discussions exists'` state. + +```ruby +Pact.provider_states_for "Merge Request Page" do + provider_state "a merge request with discussions exists" do + + end +end +``` + +### The `set_up` block + +This is where you define the test data creation steps. Use `FactoryBot` to create the data. As you create the test data, you can keep [running the provider test](index.md#run-the-provider-tests) to check on the status of the test and figure out what else is missing in your data setup. + +```ruby +Pact.provider_states_for "Merge Request Page" do + provider_state "a merge request with discussions exists" do + set_up do + user = User.find_by(name: Provider::UsersHelper::CONTRACT_USER_NAME) + namespace = create(:namespace, name: 'gitlab-org') + project = create(:project, name: 'gitlab-qa', namespace: namespace) + + project.add_maintainer(user) + + merge_request = create(:merge_request_with_diffs, id: 1, source_project: project, author: user) + + create(:discussion_note_on_merge_request, noteable: merge_request, project: project, author: user) + end + end +end +``` + +Note the `Provider::UsersHelper::CONTRACT_USER_NAME` here to fetch a user is a user that is from the [`spec_helper`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/provider/spec_helper.rb) that sets up a user before any of these tests run. + +And with that, the provider tests for `discussion_helper.rb` should now pass with this. diff --git a/doc/development/testing_guide/end_to_end/beginners_guide.md b/doc/development/testing_guide/end_to_end/beginners_guide.md index 39a3e3445ea..a13011d0101 100644 --- a/doc/development/testing_guide/end_to_end/beginners_guide.md +++ b/doc/development/testing_guide/end_to_end/beginners_guide.md @@ -343,8 +343,8 @@ Before running the spec, make sure that: To run the spec, run the following command: -```ruby -GITLAB_PASSWORD=<GDK root password> bundle exec bin/qa Test::Instance::All http://localhost:3000 -- <test_file> +```shell +GITLAB_PASSWORD=<GDK root password> bundle exec rspec <test_file> ``` Where `<test_file>` is: @@ -352,6 +352,8 @@ Where `<test_file>` is: - `qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb` when running the Login example. - `qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb` when running the Issue example. +Additional information on test execution and possible options are described in ["QA framework README"](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/README.md#run-the-end-to-end-tests-in-a-local-development-environment) + ## End-to-end test merge request template When submitting a new end-to-end test, use the ["New End to End Test"](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/New%20End%20To%20End%20Test.md) diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index bd9896934c7..85f8beeacad 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -189,9 +189,9 @@ Attach the `:aggregate_failures` metadata to the example if multiple expectation it 'searches', :aggregate_failures do Page::Search::Results.perform do |search| expect(search).to have_file_in_project(template[:file_name], project.name) - + search.switch_to_code - + expect(search).to have_file_with_content(template[:file_name], content[0..33]) end end @@ -208,6 +208,54 @@ it 'searches' do end ``` +## Avoid multiple actions in `expect do ... raise_error` blocks + +When you wrap multiple actions in a single `expect do ... end.not_to raise_error` or `expect do ... end.to raise_error` block, +it can be hard to debug the actual cause of the failure, because of how the logs are printed. Important information can be truncated +or missing altogether. + +For example, if you encapsulate some actions and expectations in a private method in the test, like `expect_owner_permissions_allow_delete_issue`: + +```ruby +it "has Owner role with Owner permissions" do + Page::Dashboard::Projects.perform do |projects| + projects.filter_by_name(project.name) + + expect(projects).to have_project_with_access_role(project.name, 'Owner') + end + + expect_owner_permissions_allow_delete_issue +end +``` + +Then, in the method itself: + +```ruby +#=> Good +def expect_owner_permissions_allow_delete_issue + issue.visit! + + Page::Project::Issue::Show.perform(&:delete_issue) + + Page::Project::Issue::Index.perform do |index| + expect(index).not_to have_issue(issue) + end +end + +#=> Bad +def expect_owner_permissions_allow_delete_issue + expect do + issue.visit! + + Page::Project::Issue::Show.perform(&:delete_issue) + + Page::Project::Issue::Index.perform do |index| + expect(index).not_to have_issue(issue) + end + end.not_to raise_error +end +``` + ## Prefer to split tests across multiple files Our framework includes a couple of parallelization mechanisms that work by executing spec files in parallel. diff --git a/doc/development/testing_guide/end_to_end/capybara_to_chemlab_migration_guide.md b/doc/development/testing_guide/end_to_end/capybara_to_chemlab_migration_guide.md index 9c7e0ef73a8..a71e076b57f 100644 --- a/doc/development/testing_guide/end_to_end/capybara_to_chemlab_migration_guide.md +++ b/doc/development/testing_guide/end_to_end/capybara_to_chemlab_migration_guide.md @@ -12,7 +12,7 @@ Given the view: ```html <form id="my-form"> - <label for="first-name">First name</label> + <label for="first-name">First name</label> <input type="text" name="first-name" data-qa-selector="first_name" /> <label for="last-name">Last name</label> @@ -26,7 +26,7 @@ Given the view: <label for="password">Password</label> <input type="password" name="password" data-qa-selector="password" /> - + <input type="submit" value="Continue" data-qa-selector="continue"/> </form> ``` diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index b4ec9e8ccd3..cb4c8e8a6e8 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -30,7 +30,7 @@ feature flag is under test. - Format: `feature_flag: { name: 'feature_flag_name', scope: :project }` - When `scope` is set to `:global`, the test will be **skipped on all live .com environments**. This is to avoid issues with feature flag changes affecting other tests or users on that environment. -- When `scope` is set to any other value (such as `:project`, `:group` or `:user`), or if no `scope` is specified, the test will only be **skipped on canary and production**. +- When `scope` is set to any other value (such as `:project`, `:group` or `:user`), or if no `scope` is specified, the test will only be **skipped on canary, production, and preprod**. This is due to the fact that administrator access is not available there. **WARNING:** You are strongly advised to first try and [enable feature flags only for a group, project, user](../../feature_flags/index.md#feature-actors), @@ -192,10 +192,26 @@ End-to-end tests should pass with a feature flag enabled before it is enabled on ### Automatic test execution when a feature flag definition changes -If a merge request adds or edits a [feature flag definition file](../../feature_flags/index.md#feature-flag-definition-and-validation), -two `package-and-qa` jobs will be included automatically in the merge request pipeline. One job will enable the defined -feature flag and the other will disable it. The jobs execute the same suite of tests to confirm that they pass with if -the feature flag is either enabled or disabled. +There are two ways to confirm that end-to-end tests pass: + +- If a merge request adds or edits a [feature flag definition file](../../feature_flags/index.md#feature-flag-definition-and-validation), + two `package-and-qa` jobs (`package-and-qa-ff-enabled` and `package-and-qa-ff-disabled`) are included automatically in the merge request + pipeline. One job enables the defined feature flag and the other job disables it. The jobs execute the same suite of tests to confirm + that they pass with the feature flag either enabled or disabled. +- In some cases, if `package-and-qa` hasn't been triggered automatically, or if it has run the tests with the default feature flag values + (which might not be desired), you can create a Draft MR that enables the feature flag to ensure that all E2E tests pass with the feature + flag enabled. + +### Troubleshooting end-to-end test failures with feature flag enabled + +If enabling the feature flag results in E2E test failures, you can browse the artifacts in the failed pipeline to see screenshots of the failed tests. After which, you can either: + +- Identify tests that need to be updated and contact the relevant [counterpart Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors) responsible for updating the tests or assisting another engineer to do so. However, if a change does not go through [quad-planning](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/quad-planning/) and a required test update is not made, test failures could block deployment. +- Run the failed tests [locally](https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa#run-the-end-to-end-tests-in-a-local-development-environment) + with the [feature flag enabled](https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa#running-tests-with-a-feature-flag-enabled-or-disabled). + This option requires considerable amount of setup, but you'll be able to see what the browser is doing as it's running the failed + tests, which can help debug the problem faster. You can also refer to the [Troubleshooting Guide for E2E tests](troubleshooting.md) for + support for common blockers. ### Test execution during feature development diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index 85ab4d479f9..c93e8c9d13f 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -150,7 +150,7 @@ In our case, `data-qa-selector="login_field"`, `data-qa-selector="password_field ```haml = f.text_field :login, class: "form-control top", autofocus: "autofocus", autocapitalize: "off", autocorrect: "off", required: true, title: "This field is required.", data: { qa_selector: 'login_field' } = f.password_field :password, class: "form-control bottom", required: true, title: "This field is required.", data: { qa_selector: 'password_field' } -= f.submit "Sign in", class: "btn btn-success", data: { qa_selector: 'sign_in_button' } += f.submit "Sign in", class: "btn btn-confirm", data: { qa_selector: 'sign_in_button' } ``` Things to note: diff --git a/doc/development/testing_guide/end_to_end/resources.md b/doc/development/testing_guide/end_to_end/resources.md index e2b005d8a1b..dacc428aec6 100644 --- a/doc/development/testing_guide/end_to_end/resources.md +++ b/doc/development/testing_guide/end_to_end/resources.md @@ -531,7 +531,7 @@ When you implement a new type of reusable resource there are two `private` metho can be validated. They are: - `reference_resource`: creates a new instance of the resource that can be compared with the one that was used during the tests. -- `unique_identifiers`: returns an array of attributes that allow the resource to be identified (e.g., name) and that are therefore +- `unique_identifiers`: returns an array of attributes that allow the resource to be identified (for example, name) and that are therefore expected to differ when comparing the reference resource with the resource reused in the tests. The following example shows the implementation of those two methods in `QA::Resource::ReusableProject`. diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index 0163f2e648c..591d03db7b8 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -15,13 +15,14 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec |-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `:elasticsearch` | The test requires an Elasticsearch service. It is used by the [instance-level scenario](https://gitlab.com/gitlab-org/gitlab-qa#definitions) [`Test::Integration::Elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/72b62b51bdf513e2936301cb6c7c91ec27c35b4d/qa/qa/ee/scenario/test/integration/elasticsearch.rb) to include only tests that require Elasticsearch. | | `:except` | The test is to be run in their typical execution contexts _except_ as specified. See [test execution context selection](execution_context_selection.md) for more information. | -| `:feature_flag` | The test uses a feature flag and therefore requires an administrator account to run. When `scope` is set to `:global`, the test will be skipped on all live .com environments. Otherwise, it will be skipped only on Canary and Production. See [testing with feature flags](../../../development/testing_guide/end_to_end/feature_flags.md) for more details. | +| `:feature_flag` | The test uses a feature flag and therefore requires an administrator account to run. When `scope` is set to `:global`, the test will be skipped on all live .com environments. Otherwise, it will be skipped only on Canary, Production, and Preprod. See [testing with feature flags](../../../development/testing_guide/end_to_end/feature_flags.md) for more details. | | `:geo` | The test requires two GitLab Geo instances - a primary and a secondary - to be spun up. | | `:gitaly_cluster` | The test runs against a GitLab instance where repositories are stored on redundant Gitaly nodes behind a Praefect node. All nodes are [separate containers](../../../administration/gitaly/praefect.md#requirements). Tests that use this tag have a longer setup time since there are three additional containers that need to be started. | | `:github` | The test requires a GitHub personal access token. | | `:group_saml` | The test requires a GitLab instance that has SAML SSO enabled at the group level. Interacts with an external SAML identity provider. Paired with the `:orchestrated` tag. | | `:instance_saml` | The test requires a GitLab instance that has SAML SSO enabled at the instance level. Interacts with an external SAML identity provider. Paired with the `:orchestrated` tag. | | `:integrations` | This aims to test the available [integrations](../../../user/project/integrations/index.md#available-integrations). The test requires Docker to be installed in the run context. It will provision the containers and can be run against a local instance or using the `gitlab-qa` scenario `Test::Integration::Integrations` | +| `:issue`, `:issue_${num}` | Optional links to issues which might be related to the spec. Helps keep track of related issues and can also be used by tools that create test reports. Currently added automatically to `Allure` test report. Multiple tags can be used by adding an optional numeric suffix like `issue_1`, `issue_2` etc. | | `:service_ping_disabled` | The test interacts with the GitLab configuration service ping at the instance level to turn Admin Area setting service ping checkbox on or off. This tag will have the test run only in the `service_ping_disabled` job and must be paired with the `:orchestrated` and `:requires_admin` tags. | | `:jira` | The test requires a Jira Server. [GitLab-QA](https://gitlab.com/gitlab-org/gitlab-qa) provisions the Jira Server in a Docker container when the `Test::Integration::Jira` test scenario is run. | | `:kubernetes` | The test includes a GitLab instance that is configured to be run behind an SSH tunnel, allowing a TLS-accessible GitLab. This test also includes provisioning of at least one Kubernetes cluster to test against. _This tag is often be paired with `:orchestrated`._ | @@ -42,6 +43,7 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:requires_git_protocol_v2` | The test requires that Git protocol version 2 is enabled on the server. It's assumed to be enabled by default but if not the test can be skipped by setting `QA_CAN_TEST_GIT_PROTOCOL_V2` to `false`. | | `:requires_praefect` | The test requires that the GitLab instance uses [Gitaly Cluster](../../../administration/gitaly/praefect.md) (a.k.a. Praefect) as the repository storage . It's assumed to be used by default but if not the test can be skipped by setting `QA_CAN_TEST_PRAEFECT` to `false`. | | `:runner` | The test depends on and sets up a GitLab Runner instance, typically to run a pipeline. | +| `:sanity_feature_flags` | The test verifies the functioning of the feature flag handling part of the test framework | | `:skip_live_env` | The test is excluded when run against live deployed environments such as Staging, Canary, and Production. | | `:skip_fips_env` | The test is excluded when run against an environment in FIPS mode. | | `:skip_signup_disabled` | The test uses UI to sign up a new user and is skipped in any environment that does not allow new user registration via the UI. | @@ -49,4 +51,3 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:smtp` | The test requires a GitLab instance to be configured to use an SMTP server. Tests SMTP notification email delivery from GitLab by using MailHog. | | `:testcase` | The link to the test case issue in the [GitLab Project Test Cases](https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases). | | `:transient` | The test tests transient bugs. It is excluded by default. | -| `:issue`, `:issue_${num}` | Optional links to issues which might be related to the spec. Helps keep track of related issues and can also be used by tools that create test reports. Currently added automatically to `Allure` test report. Multiple tags can be used by adding an optional numeric suffix like `issue_1`, `issue_2` etc. | diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index 599e1104b72..438294161ac 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -153,7 +153,7 @@ Examples of tests which require a runner: Example: ```shell -docker run \ +docker run \ --detach \ --hostname interface_ip_address \ --publish 80:80 \ @@ -274,7 +274,7 @@ Geo requires an EE license. To visit the Geo sites in your browser, you need a r 1. To run end-to-end tests from your local GDK, run the [`EE::Scenario::Test::Geo` scenario](https://gitlab.com/gitlab-org/gitlab/-/blob/f7272b77e80215c39d1ffeaed27794c220dbe03f/qa/qa/ee/scenario/test/geo.rb) from the [`gitlab/qa/` directory](https://gitlab.com/gitlab-org/gitlab/-/blob/f7272b77e80215c39d1ffeaed27794c220dbe03f/qa). Include `--without-setup` to skip the Geo configuration steps. ```shell - QA_DEBUG=true GITLAB_QA_ACCESS_TOKEN=[add token here] GITLAB_QA_ADMIN_ACCESS_TOKEN=[add token here] bundle exec bin/qa QA::EE::Scenario::Test::Geo \ + QA_LOG_LEVEL=debug GITLAB_QA_ACCESS_TOKEN=[add token here] GITLAB_QA_ADMIN_ACCESS_TOKEN=[add token here] bundle exec bin/qa QA::EE::Scenario::Test::Geo \ --primary-address http://gitlab-primary.geo \ --secondary-address http://gitlab-secondary.geo \ --without-setup @@ -283,7 +283,7 @@ Geo requires an EE license. To visit the Geo sites in your browser, you need a r If the containers need to be configured first (for example, if you used the `--no-tests` option in the previous step), run the `QA::EE::Scenario::Test::Geo scenario` as shown below to first do the Geo configuration steps, and then run Geo end-to-end tests. Make sure that `EE_LICENSE` is (still) defined in your shell session. ```shell - QA_DEBUG=true bundle exec bin/qa QA::EE::Scenario::Test::Geo \ + QA_LOG_LEVEL=debug bundle exec bin/qa QA::EE::Scenario::Test::Geo \ --primary-address http://gitlab-primary.geo \ --primary-name gitlab-primary \ --secondary-address http://gitlab-secondary.geo \ @@ -354,7 +354,7 @@ To run the LDAP tests on your local with TLS enabled, follow these steps: 1. Run an LDAP test from [`gitlab/qa`](https://gitlab.com/gitlab-org/gitlab/-/tree/d5447ebb5f99d4c72780681ddf4dc25b0738acba/qa) directory: ```shell - GITLAB_LDAP_USERNAME="tanuki" GITLAB_LDAP_PASSWORD="password" QA_DEBUG=true WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All https://gitlab.test qa/specs/features/browser_ui/1_manage/login/log_into_gitlab_via_ldap_spec.rb + GITLAB_LDAP_USERNAME="tanuki" GITLAB_LDAP_PASSWORD="password" QA_LOG_LEVEL=debug WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All https://gitlab.test qa/specs/features/browser_ui/1_manage/login/log_into_gitlab_via_ldap_spec.rb ``` ### Running LDAP tests with TLS disabled @@ -382,7 +382,7 @@ To run the LDAP tests on your local with TLS disabled, follow these steps: 1. Run an LDAP test from [`gitlab/qa`](https://gitlab.com/gitlab-org/gitlab/-/tree/d5447ebb5f99d4c72780681ddf4dc25b0738acba/qa) directory: ```shell - GITLAB_LDAP_USERNAME="tanuki" GITLAB_LDAP_PASSWORD="password" QA_DEBUG=true WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All http://localhost qa/specs/features/browser_ui/1_manage/login/log_into_gitlab_via_ldap_spec.rb + GITLAB_LDAP_USERNAME="tanuki" GITLAB_LDAP_PASSWORD="password" QA_LOG_LEVEL=debug WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All http://localhost qa/specs/features/browser_ui/1_manage/login/log_into_gitlab_via_ldap_spec.rb ``` ## Guide to the mobile suite diff --git a/doc/development/testing_guide/end_to_end/troubleshooting.md b/doc/development/testing_guide/end_to_end/troubleshooting.md index 951fb056a4c..76d19dc0159 100644 --- a/doc/development/testing_guide/end_to_end/troubleshooting.md +++ b/doc/development/testing_guide/end_to_end/troubleshooting.md @@ -25,12 +25,12 @@ WEBDRIVER_HEADLESS=false bundle exec bin/qa Test::Instance::All http://localhost Sometimes a test might fail and the failure stack trace doesn't provide enough information to determine what went wrong. You can get more information by enabling -debug logs by setting `QA_DEBUG=true`, to see what the test framework is attempting. +debug logs by setting `QA_LOG_LEVEL=debug`, to see what the test framework is attempting. For example: ```shell cd gitlab/qa -QA_DEBUG=true bundle exec bin/qa Test::Instance::All http://localhost:3000 +QA_LOG_LEVEL=debug bundle exec bin/qa Test::Instance::All http://localhost:3000 ``` The test framework then outputs many logs showing the actions taken during diff --git a/doc/development/testing_guide/index.md b/doc/development/testing_guide/index.md index 2e00a00c454..fa9f1f1ac3e 100644 --- a/doc/development/testing_guide/index.md +++ b/doc/development/testing_guide/index.md @@ -70,4 +70,8 @@ Everything you should know about how to run end-to-end tests using Everything you should know about how to test migrations. +## [Contract tests](contract/index.md) + +Introduction to contract testing, how to run the tests, and how to write them. + [Return to Development documentation](../index.md) diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index ff4b77dec2c..f1083c23406 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -6,13 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Review Apps -Review Apps are deployed using the `start-review-app-pipeline` job. This job triggers a child pipeline containing a series of jobs to perform the various tasks needed to deploy a Review App. +Review Apps are deployed using the `start-review-app-pipeline` job which triggers a child pipeline containing a series of jobs to perform the various tasks needed to deploy a Review App.  For any of the following scenarios, the `start-review-app-pipeline` job would be automatically started: -- for merge requests with CI config changes +- for merge requests with CI configuration changes - for merge requests with frontend changes - for merge requests with changes to `{,ee/,jh/}{app/controllers}/**/*` - for merge requests with changes to `{,ee/,jh/}{app/models}/**/*` @@ -27,6 +27,8 @@ On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) in `review` stage), the `review-qa-smoke` and `review-qa-reliable` jobs are automatically started. The `review-qa-smoke` runs the QA smoke suite and the `review-qa-reliable` executes E2E tests identified as [reliable](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/reliable-tests). +`review-qa-*` jobs ensure that end-to-end tests for the changes in the merge request pass in a live environment. This shifts the identification of e2e failures from an environment on the path to production to the merge request, to prevent breaking features on GitLab.com or costly GitLab.com deployment blockers. `review-qa-*` failures should be investigated with counterpart SET involvement if needed to help determine the root cause of the error. + You can also manually start the `review-qa-all`: it runs the full QA suite. After the end-to-end test runs have finished, [Allure reports](https://github.com/allure-framework/allure2) are generated and published by @@ -34,6 +36,10 @@ the `allure-report-qa-smoke`, `allure-report-qa-reliable`, and `allure-report-qa Errors can be found in the `gitlab-review-apps` Sentry project and [filterable by Review App URL](https://sentry.gitlab.net/gitlab/gitlab-review-apps/?query=url%3A%22https%3A%2F%2Fgitlab-review-require-ve-u92nn2.gitlab-review.app%2F%22) or [commit SHA](https://sentry.gitlab.net/gitlab/gitlab-review-apps/releases/6095b501da7/all-events/). +### Bypass failed review app deployment to merge a broken `master` fix + +Maintainers can elect to use the [process for merging during broken `master`](https://about.gitlab.com/handbook/engineering/workflow/#instructions-for-the-maintainer) if a customer-critical merge request is blocked by pipelines failing due to review app deployment failures. + ## Performance Metrics On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) in the `qa` stage, the @@ -94,8 +100,8 @@ the GitLab handbook information for the [shared 1Password account](https://about 1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) and the `container.pods.exec` permission first. 1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps). For example, `review-qa-raise-e-12chm0`. 1. Find and open the `toolbox` Deployment. For example, `review-qa-raise-e-12chm0-toolbox`. -1. Click on the Pod in the "Managed pods" section. For example, `review-qa-raise-e-12chm0-toolbox-d5455cc8-2lsvz`. -1. Click on the `KUBECTL` dropdown, then `Exec` -> `toolbox`. +1. Select the Pod in the "Managed pods" section. For example, `review-qa-raise-e-12chm0-toolbox-d5455cc8-2lsvz`. +1. Select the `KUBECTL` dropdown, then `Exec` -> `toolbox`. 1. Replace `-c toolbox -- ls` with `-it -- gitlab-rails console` from the default command or - Run `kubectl exec --namespace review-qa-raise-e-12chm0 review-qa-raise-e-12chm0-toolbox-d5455cc8-2lsvz -it -- gitlab-rails console` and @@ -107,8 +113,8 @@ the GitLab handbook information for the [shared 1Password account](https://about 1. Make sure you [have access to the cluster](#get-access-to-the-gcp-review-apps-cluster) and the `container.pods.getLogs` permission first. 1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps). For example, `review-qa-raise-e-12chm0`. 1. Find and open the `migrations` Deployment. For example, `review-qa-raise-e-12chm0-migrations.1`. -1. Click on the Pod in the "Managed pods" section. For example, `review-qa-raise-e-12chm0-migrations.1-nqwtx`. -1. Click on the `Container logs` link. +1. Select the Pod in the "Managed pods" section. For example, `review-qa-raise-e-12chm0-migrations.1-nqwtx`. +1. Select `Container logs`. Alternatively, you could use the [Logs Explorer](https://console.cloud.google.com/logs/query;query=?project=gitlab-review-apps) which provides more utility to search logs. An example query for a pod name is as follows: @@ -199,7 +205,7 @@ subgraph "CNG-mirror pipeline" issue with a link to your merge request. Note that the deployment failure can reveal an actual problem introduced in your merge request (that is, this isn't necessarily a transient failure)! -- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing, +- If the `review-qa-smoke` or `review-qa-reliable` job keeps failing (note that we already retry them once), please check the job's logs: you could discover an actual problem introduced in your merge request. You can also download the artifacts to see screenshots of the page at the time the failures occurred. If you don't find the cause of the @@ -237,7 +243,7 @@ due to this [known issue on the Kubernetes executor for GitLab Runner](https://g ### Helm The Helm version used is defined in the -[`registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14` image](https://gitlab.com/gitlab-org/gitlab-build-images/-/blob/master/Dockerfile.gitlab-helm3-kubectl1.14#L7) +[`registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17` image](https://gitlab.com/gitlab-org/gitlab-build-images/-/blob/master/Dockerfile.gitlab-helm3.5-kubectl1.17#L6) used by the `review-deploy` and `review-stop` jobs. ## Diagnosing unhealthy Review App releases diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index 9ca2d0db93c..02f32a031dc 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -55,7 +55,6 @@ records should use stubs/doubles as much as possible. | `lib/` | `spec/lib/` | RSpec | | | `lib/tasks/` | `spec/tasks/` | RSpec | | | `rubocop/` | `spec/rubocop/` | RSpec | | -| `spec/factories` | `spec/factories_spec.rb` | RSpec | | ### Frontend unit tests diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index e06ece38135..17fcd5b3e88 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Understanding EXPLAIN plans PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This -command can be invaluable when trying to determine how a query will perform. +command can be invaluable when trying to determine how a query performs. You can use this command directly in your SQL query, as long as the query starts with it: @@ -26,7 +26,7 @@ Aggregate (cost=922411.76..922411.77 rows=1 width=8) Filter: (visibility_level = ANY ('{0,20}'::integer[])) ``` -When using _just_ `EXPLAIN`, PostgreSQL won't actually execute our query, +When using _just_ `EXPLAIN`, PostgreSQL does not actually execute our query, instead it produces an _estimated_ execution plan based on the available statistics. This means the actual plan can differ quite a bit. Fortunately, PostgreSQL provides us with the option to execute the query as well. To do so, @@ -39,7 +39,7 @@ FROM projects WHERE visibility_level IN (0, 20); ``` -This will produce: +This produces: ```sql Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) @@ -54,7 +54,7 @@ As we can see this plan is quite different, and includes a lot more data. Let's discuss this step by step. Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a -query that will write data or might time out. If the query modifies data, +query that writes data or might time out. If the query modifies data, consider wrapping it in a transaction that rolls back automatically like so: ```sql @@ -73,7 +73,7 @@ FROM projects WHERE visibility_level IN (0, 20); ``` -This will then produce: +This then produces: ```sql Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) @@ -120,10 +120,10 @@ Aggregate (cost=922411.76..922411.77 rows=1 width=8) Here the first node executed is `Seq scan on projects`. The `Filter:` is an additional filter applied to the results of the node. A filter is very similar to Ruby's `Array#select`: it takes the input rows, applies the filter, and -produces a new list of rows. Once the node is done, we perform the `Aggregate` +produces a new list of rows. After the node is done, we perform the `Aggregate` above it. -Nested nodes will look like this: +Nested nodes look like this: ```sql Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) @@ -152,7 +152,7 @@ number of rows produced, the number of loops performed, and more. For example: Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) ``` -Here we can see that our cost ranges from `0.00..908044.47` (we'll cover this in +Here we can see that our cost ranges from `0.00..908044.47` (we cover this in a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` statistics describes the estimated width of each row, in bytes. @@ -171,7 +171,7 @@ The startup cost states how expensive it was to start the node, with the total cost describing how expensive the entire node was. In general: the greater the values, the more expensive the node. -When using `EXPLAIN ANALYZE`, these statistics will also include the actual time +When using `EXPLAIN ANALYZE`, these statistics also include the actual time (in milliseconds) spent, and other runtime statistics (for example, the actual number of produced rows): @@ -183,7 +183,7 @@ Here we can see we estimated 5,746,969 rows to be returned, but in reality we returned 5,746,940 rows. We can also see that _just_ this sequential scan took 2.98 seconds to run. -Using `EXPLAIN (ANALYZE, BUFFERS)` will also give us information about the +Using `EXPLAIN (ANALYZE, BUFFERS)` also gives us information about the number of rows removed by a filter, the number of buffers used, and more. For example: @@ -242,7 +242,7 @@ retrieving lots of rows, so it's best to avoid these for large tables. A scan on an index that did not require fetching anything from the table. In certain cases an index only scan may still fetch data from the table, in this -case the node will include a `Heap Fetches:` statistic. +case the node includes a `Heap Fetches:` statistic. ### Index Scan @@ -273,7 +273,7 @@ Sorts the input rows as specified using an `ORDER BY` statement. ### Nested Loop -A nested loop will execute its child nodes for every row produced by a node that +A nested loop executes its child nodes for every row produced by a node that precedes it. For example: ```sql @@ -316,7 +316,7 @@ FROM users WHERE twitter != ''; ``` -This will produce the following plan: +This produces the following plan: ```sql Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) @@ -435,7 +435,7 @@ This index would only index the `email` value of rows that match `WHERE id < CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; ``` -Once created, if we run our query again we will be given the following plan: +After being created, if we run our query again we are given the following plan: ```sql Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) @@ -466,7 +466,7 @@ be used for comparison (for example, it depends a lot on the state of cache). When optimizing a query, we usually need to reduce the amount of data we're dealing with. Indexes are the way to work with fewer pages (buffers) to get the result, so, during optimization, look at the number of buffers used (read and hit), -and work on reducing these numbers. Reduced timing will be the consequence of reduced +and work on reducing these numbers. Reduced timing is the consequence of reduced buffer numbers. [Database Lab Engine](#database-lab-engine) guarantees that the plan is structurally identical to production (and overall number of buffers is the same as on production), but difference in cache state and I/O speed may lead to different timings. @@ -508,8 +508,8 @@ index on `projects.visibility_level` to somehow turn this Sequential scan + filter into an index-only scan. Unfortunately, doing so is unlikely to improve anything. Contrary to what some -might believe, an index being present _does not guarantee_ that PostgreSQL will -actually use it. For example, when doing a `SELECT * FROM projects` it is much +might believe, an index being present _does not guarantee_ that PostgreSQL +actually uses it. For example, when doing a `SELECT * FROM projects` it is much cheaper to just scan the entire table, instead of using an index and then fetching data from the table. In such cases PostgreSQL may decide to not use an index. @@ -539,7 +539,7 @@ For GitLab.com this produces: Here the total number of projects is 5,811,804, and 5,746,126 of those are of level 0 or 20. That's 98% of the entire table! -So no matter what we do, this query will retrieve 98% of the entire table. Since +So no matter what we do, this query retrieves 98% of the entire table. Since most time is spent doing exactly that, there isn't really much we can do to improve this query, other than _not_ running it at all. @@ -589,7 +589,7 @@ Foreign-key constraints: "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE ``` -Let's rewrite our query to JOIN this table onto our projects, and get the +Let's rewrite our query to `JOIN` this table onto our projects, and get the projects for a specific user: ```sql @@ -604,7 +604,7 @@ AND user_interacted_projects.user_id = 1; What we do here is the following: 1. Get our projects. -1. INNER JOIN `user_interacted_projects`, meaning we're only left with rows in +1. `INNER JOIN` `user_interacted_projects`, meaning we're only left with rows in `projects` that have a corresponding row in `user_interacted_projects`. 1. Limit this to the projects with `visibility_level` of 0 or 20, and to projects that the user with ID 1 interacted with. @@ -765,7 +765,7 @@ The web interface comes with the following execution plan visualizers included: #### Tips & Tricks -The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings will be applied to all subsequent commands until you reset them. For example you can disable parallel queries with +The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings are applied to all subsequent commands until you reset them. For example you can disable parallel queries with ```sql exec SET max_parallel_workers_per_gather = 0 diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md index 4e907530a9f..d44f2f69168 100644 --- a/doc/development/uploads/working_with_uploads.md +++ b/doc/development/uploads/working_with_uploads.md @@ -101,23 +101,24 @@ Therefore, document new uploads here by slotting them into the following tables: | Pipeline artifacts | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/pipelines/<pipeline_id>/artifacts/<artifact_id>` | | Live job traces | `fog` | `sidekiq` | `/artifacts/tmp/builds/<job_id>/chunks/<chunk_index>.log` | | Job traces archive | `carrierwave` | `sidekiq` | `/artifacts/<proj_id_hash>/<date>/<job_id>/<artifact_id>/job.log` | -| Autoscale runner caching | N/A | `gitlab-runner` | `/gitlab-com-[platform-]runners-cache/???` | -| Backups | N/A | `s3cmd`, `awscli`, or `gcs` | `/gitlab-backups/???` | -| Git LFS | `direct upload` | `workhorse` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | +| Autoscale runner caching | Not applicable | `gitlab-runner` | `/gitlab-com-[platform-]runners-cache/???` | +| Backups | Not applicable | `s3cmd`, `awscli`, or `gcs` | `/gitlab-backups/???` | +| Git LFS | `direct upload` | `workhorse` | `/lfs-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | | Design management files | `disk buffering` | `rails controller` | `/lsf-objects/<lfs_obj_oid[0:2]>/<lfs_obj_oid[2:2]>` | -| Design management thumbnails | `carrierwave` | `sidekiq` | `/uploads/design_management/action/image_v432x230/<model_id>` | +| Design management thumbnails | `carrierwave` | `sidekiq` | `/uploads/design_management/action/image_v432x230/<model_id>/<original_lfs_obj_oid[2:2]` | | Generic file uploads | `direct upload` | `workhorse` | `/uploads/@hashed/[0:2]/[2:4]/<hash1>/<hash2>/file` | | Generic file uploads - personal snippets | `direct upload` | `workhorse` | `/uploads/personal_snippet/<snippet_id>/<filename>` | | Global appearance settings | `disk buffering` | `rails controller` | `/uploads/appearance/...` | | Topics | `disk buffering` | `rails controller` | `/uploads/projects/topic/...` | | Avatar images | `direct upload` | `workhorse` | `/uploads/[user,group,project]/avatar/<model_id>` | -| Import/export | `direct upload` | `workhorse` | `/uploads/import_export_upload/???` | +| Import | `direct upload` | `workhorse` | `/uploads/import_export_upload/import_file/<model_id>/<file_name>` | +| Export | `carrierwave` | `sidekiq` | `/uploads/import_export_upload/export_file/<model_id>/<timestamp>_<namespace>-<project_name>_export.tag.gz` | | GitLab Migration | `carrierwave` | `sidekiq` | `/uploads/bulk_imports/???` | | MR diffs | `carrierwave` | `sidekiq` | `/external-diffs/merge_request_diffs/mr-<mr_id>/diff-<diff_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<proj_id_hash>/packages/<pkg_segment>/files/<pkg_file_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_component_file/<component_file_id>` | -| Package manager archives | `direct upload` | `sidekiq` | `/packages/<container_id_hash>/debian_*_distribution/<distribution_file_id>` | -| Container image cache (?) | `direct upload` | `workhorse` | `/dependency-proxy/<group_id_hash>/dependency_proxy/<group_id>/files/<proxy_id>/<blob_id or manifest_id>` | +| [Package manager assets (except for NPM)](../../user/packages/package_registry/index.md) | `direct upload` | `workhorse` | `/packages/<proj_id_hash>/packages/<package_id>/files/<package_file_id>` | +| [NPM Package manager assets](../../user/packages/npm_registry/index.md) | `carrierwave` | `grape API` | `/packages/<proj_id_hash>/packages/<package_id>/files/<package_file_id>` | +| [Debian Package manager assets](../../user/packages/debian_repository/index.md) | `direct upload` | `workhorse` | `/packages/<group_id or project_id_hash>/debian_*/<group_id or project_id or distribution_file_id>` | +| [Dependency Proxy cache](../../user/packages/dependency_proxy/index.md) | [`send_dependency`](https://gitlab.com/gitlab-org/gitlab/-/blob/6ed73615ff1261e6ed85c8f57181a65f5b4ffada/workhorse/internal/dependencyproxy/dependencyproxy.go) | `workhorse` | `/dependency-proxy/<group_id_hash>/dependency_proxy/<group_id>/files/<blob_id or manifest_id>` | | Terraform state files | `carrierwave` | `rails controller` | `/terraform/<proj_id_hash>/<terraform_state_id>` | | Pages content archives | `carrierwave` | `sidekiq` | `/gitlab-gprd-pages/<proj_id_hash>/pages_deployments/<deployment_id>/` | | Secure Files | `carrierwave` | `sidekiq` | `/ci-secure-files/<proj_id_hash>/secure_files/<secure_file_id>/` | diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index bda9c68eae5..55347edf4ec 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -34,5 +34,5 @@ to be wrapped in a `Gitlab::Database.read_only?` or `Gitlab::Database.read_write guard, to make sure it doesn't for read-only databases. We have a Rails Middleware that filters any potentially writing -operations (the CUD operations of CRUD) and prevent the user from trying +operations (the `CUD` operations of CRUD) and prevent the user from trying to update the database and getting a 500 error (see `Gitlab::Middleware::ReadOnly`). diff --git a/doc/development/windows.md b/doc/development/windows.md index fb095b68939..3eed9c057ab 100644 --- a/doc/development/windows.md +++ b/doc/development/windows.md @@ -65,21 +65,21 @@ Build a Google Cloud image with the above shared runners repository by doing the 1. In a web browser, go to the [Google Cloud Platform console](https://console.cloud.google.com/compute/images). 1. Filter images by the name you used when creating image, `windows` is likely all you need to filter by. -1. Click the image's name. -1. Click the **CREATE INSTANCE** link. +1. Select the image's name. +1. Select **CREATE INSTANCE**. 1. Important: Change name to what you'd like as you can't change it later. 1. Optional: Change Region to be closest to you as well as any other option you'd like. -1. Click **Create** at the bottom of the page. -1. Click the name of your newly created VM Instance (optionally you can filter to find it). -1. Click **Set Windows password**. +1. Select **Create** at the bottom of the page. +1. Select the name of your newly created VM Instance (optionally you can filter to find it). +1. Select **Set Windows password**. 1. Optional: Set a username or use default. -1. Click **Next**. +1. Select **Next**. 1. Copy and save the password as it is not shown again. -1. Click **RDP** down arrow. -1. Click **Download the RDP file**. +1. Select **RDP** down arrow. +1. Select **Download the RDP file**. 1. Open the downloaded RDP file with the Windows remote desktop app (<https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients>). -1. Click **Continue** to accept the certificate. -1. Enter the password and click **Next**. +1. Select **Continue** to accept the certificate. +1. Enter the password and select **Next**. You should now be connected into a Windows machine with a command prompt. diff --git a/doc/development/work_items.md b/doc/development/work_items.md index d4a1073461a..9a17a152525 100644 --- a/doc/development/work_items.md +++ b/doc/development/work_items.md @@ -45,12 +45,24 @@ Here are some problems with current issues usage and why we are looking into wor - Codebase maintainability and feature development becomes a bigger challenges as we grow issues beyond its core role of issue tracking into supporting the different types and subtle differences between them. -## Work item and work item type terms +## Work item terminology -Using the terms "issue" or "issuable" to reference the types of collaboration objects -(for example, issue, bug, feature, or epic) often creates confusion. To avoid confusion, we will use the term -work item type (WIT) when referring to the type of a collaboration object. -An instance of a WIT is a work item (WI). For example, `issue#123`, `bug#456`, `requirement#789`. +To avoid confusion and ensure communication is efficient, we will use the following terms exclusively when discussing work items. + +| Term | Description | Example of misuse | Should be | +| --- | --- | --- | --- | +| work item type | Classes of work item; for example: issue, requirement, test case, incident, or task | _Epics will eventually become issues_ | _Epics will eventually become a **work item type**_ | +| work item | An instance of a work item type | | | +| work item view | The new frontend view that renders work items of any type | | | +| legacy issue view | The existing view used to render issues and incidents | | | +| issue | The existing issue model | | | +| issuable | Any model currently using the issueable module (issues, epics and MRs) | _Incidents are an **issuable**_ | _Incidents are a **work item type**_ | + +Some terms have been used in the past but have since become confusing and are now discouraged. + +| Term | Description | Example of misuse | Should be | +| --- | --- | --- | --- | +| issue type | A former way to refer to classes of work item | _Tasks are an **issue type**_ | _Tasks are a **work item type**_ | ### Migration strategy diff --git a/doc/development/workhorse/configuration.md b/doc/development/workhorse/configuration.md index ce80a155489..b86bb824ea1 100644 --- a/doc/development/workhorse/configuration.md +++ b/doc/development/workhorse/configuration.md @@ -21,47 +21,49 @@ Add any new Workhorse configuration options into the configuration file. Options: -apiCiLongPollingDuration duration - Long polling duration for job requesting for runners (default 50ns) + Long polling duration for job requesting for runners (default 50ns) -apiLimit uint - Number of API requests allowed at single time + Number of API requests allowed at single time -apiQueueDuration duration - Maximum queueing duration of requests (default 30s) + Maximum queueing duration of requests (default 30s) -apiQueueLimit uint - Number of API requests allowed to be queued + Number of API requests allowed to be queued -authBackend string - Authentication/authorization backend (default "http://localhost:8080") + Authentication/authorization backend (default "http://localhost:8080") -authSocket string - Optional: Unix domain socket to dial authBackend at + Optional: Unix domain socket to dial authBackend at -cableBackend string - Optional: ActionCable backend (default authBackend) + ActionCable backend -cableSocket string - Optional: Unix domain socket to dial cableBackend at (default authSocket) + Optional: Unix domain socket to dial cableBackend at -config string - TOML file to load config from + TOML file to load config from -developmentMode - Allow the assets to be served from Rails app + Allow the assets to be served from Rails app -documentRoot string - Path to static files content (default "public") + Path to static files content (default "public") -listenAddr string - Listen address for HTTP server (default "localhost:8181") + Listen address for HTTP server (default "localhost:8181") -listenNetwork string - Listen 'network' (tcp, tcp4, tcp6, unix) (default "tcp") + Listen 'network' (tcp, tcp4, tcp6, unix) (default "tcp") -listenUmask int - Umask for Unix socket + Umask for Unix socket -logFile string - Log file location + Log file location -logFormat string - Log format to use defaults to text (text, json, structured, none) (default "text") + Log format to use defaults to text (text, json, structured, none) (default "text") -pprofListenAddr string - pprof listening address, e.g. 'localhost:6060' + pprof listening address, e.g. 'localhost:6060' -prometheusListenAddr string - Prometheus listening address, e.g. 'localhost:9229' + Prometheus listening address, e.g. 'localhost:9229' + -propagateCorrelationID X-Request-ID + Reuse existing Correlation-ID from the incoming request header X-Request-ID if present -proxyHeadersTimeout duration - How long to wait for response headers when proxying the request (default 5m0s) + How long to wait for response headers when proxying the request (default 5m0s) -secretPath string - File with secret key to authenticate with authBackend (default "./.gitlab_workhorse_secret") + File with secret key to authenticate with authBackend (default "./.gitlab_workhorse_secret") -version - Print version and exit + Print version and exit ``` The 'auth backend' refers to the GitLab Rails application. The name is @@ -70,7 +72,7 @@ HTTP. GitLab Workhorse can listen on either a TCP or a Unix domain socket. It can also open a second listening TCP listening socket with the Go -[`net/http/pprof` profiler server](http://golang.org/pkg/net/http/pprof/). +[`net/http/pprof` profiler server](https://pkg.go.dev/net/http/pprof). GitLab Workhorse can listen on Redis build and runner registration events if you pass a valid TOML configuration file through the `-config` flag. @@ -147,6 +149,19 @@ addr = "localhost:3443" The `certificate` file should contain the concatenation of the server's certificate, any intermediates, and the CA's certificate. +Metrics endpoints can be configured similarly: + +```toml +[metrics_listener] +network = "tcp" +addr = "localhost:9229" +[metrics_listener.tls] + certificate = "/path/to/certificate" + key = "/path/to/private/key" + min_version = "tls1.2" + max_version = "tls1.3" +``` + ## Interaction of authBackend and authSocket The interaction between `authBackend` and `authSocket` can be confusing. @@ -213,6 +228,53 @@ configuration with the `GITLAB_TRACING` environment variable, like this: GITLAB_TRACING=opentracing://jaeger ./gitlab-workhorse ``` +### Propagate correlation IDs + +When a user makes an HTTP request, such as creating a new project, the +initial request is routed through Workhorse to another service, which +may in turn, make other requests. To help trace the request as it flows +across services, Workhorse generates a random value called a +[correlation ID](../../administration/troubleshooting/tracing_correlation_id.md). +Workhorse sends this correlation ID via the `X-Request-Id` HTTP header. + +Some GitLab services, such as GitLab Shell, generate their own +correlation IDs. In addition, other services, such as Gitaly, make +internal API calls that pass along a correlation ID from the original +request. In either case, the correlation ID is also passed via the +`X-Request-Id` HTTP header. + +By default, Workhorse ignores this header and always generates a new +correlation ID. This makes debugging harder and prevents distributed +tracing from working properly, since the new correlation ID is +completely unrelated to the original one. + +Workhorse can be configured to propagate an incoming correlation ID via +the `-propagateCorrelationID` command-line flag. It is highly +recommended that this option be used with an IP allow list to ensure +arbitrary values cannot be generated by untrusted clients. + +An IP allow list is specified via the `trusted_cidrs_for_propagation` +option in the Workhorse configuration file. Specify a list of CIDR blocks +that can be trusted. For example: + +```toml +trusted_cidrs_for_propagation = ["10.0.0.0/8", "127.0.0.1/32"] +``` + +NOTE: +The `-propagateCorrelationID` flag must be used for the `trusted_cidrs_for_propagation` option to work. + +### Trusted proxies + +If Workhorse is behind a reverse proxy such as NGINX, the +`trusted_cidrs_for_x_forwarded_for` option is needed to specify which +CIDR blocks can be used to trust to provide the originating IP address +via the `X-Forwarded-For` HTTP header. For example: + +```toml +trusted_cidrs_for_x_forwarded_for = ["10.0.0.0/8", "127.0.0.1/32"] +``` + ## Continuous profiling Workhorse supports continuous profiling through [LabKit](https://gitlab.com/gitlab-org/labkit/) diff --git a/doc/development/workhorse/gitlab_features.md b/doc/development/workhorse/gitlab_features.md index 2aa8d9d2399..365cc7991d8 100644 --- a/doc/development/workhorse/gitlab_features.md +++ b/doc/development/workhorse/gitlab_features.md @@ -53,14 +53,14 @@ memory than it costs to have Workhorse look after it. for example, JavaScript files and CSS files are served straight from disk. - Workhorse can modify responses sent by Rails: for example if you use - `send_file` in Rails then GitLab Workhorse will open the file on + `send_file` in Rails then GitLab Workhorse opens the file on disk and send its contents as the response body to the client. - Workhorse can take over requests after asking permission from Rails. Example: handling `git clone`. - Workhorse can modify requests before passing them to Rails. Example: when handling a Git LFS upload Workhorse first asks permission from - Rails, then it stores the request body in a tempfile, then it sends - a modified request containing the tempfile path to Rails. + Rails, then it stores the request body in a temporary file, then it sends + a modified request containing the file path to Rails. - Workhorse can manage long-lived WebSocket connections for Rails. Example: handling the terminal websocket for environments. - Workhorse does not connect to PostgreSQL, only to Rails and (optionally) Redis. diff --git a/doc/downgrade_ee_to_ce/index.md b/doc/downgrade_ee_to_ce/index.md index 865d60fed73..ca8d6f87809 100644 --- a/doc/downgrade_ee_to_ce/index.md +++ b/doc/downgrade_ee_to_ce/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -87,7 +87,7 @@ GitLab installation to the Community Edition. To downgrade a source installation, you must replace the current remote of your GitLab installation with the Community Edition's remote. After that, you can fetch the latest changes, and checkout the latest stable branch: - + ```shell git remote set-url origin git@gitlab.com:gitlab-org/gitlab-foss.git git fetch --all diff --git a/doc/drawers/advanced_search_syntax.md b/doc/drawers/advanced_search_syntax.md new file mode 100644 index 00000000000..c48efbc120a --- /dev/null +++ b/doc/drawers/advanced_search_syntax.md @@ -0,0 +1,41 @@ +--- +stage: Data Stores +group: Global Search +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" +type: drawer +source: /doc/user/search/global_search/advanced_search_syntax.md +--- + +# Search tips + +<!-- markdownlint-disable --> + +| Use | Description | Example | +|------|-------------|---------| +| `"` | Exact search | [`"gem sidekiq"`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=%22gem+sidekiq%22) | +| <code>|</code> | Or | [<code>display | banner</code>](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+%7C+banner) | +| `+` | And | [`display +banner`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=display+%2Bbanner&snippets=) | +| `-` | Exclude | [`display -banner`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+-banner) | +| `*` | Partial | [`bug error 50*`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=bug+error+50%2A&snippets=) | +| `\` | Escape | [`\*md`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=%5C*md&group_id=9970&project_id=278964) | + +## Code search + +| Use | Description | Example | +|------|-------------|---------| +| `filename:` | Filename | [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | +| `path:` | Repository location | [`path:spec/workers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fworkers&snippets=) | +| `extension:` | File extension, without the `.` | [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | +| `blob:` | Git object ID | [`blob:998707*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707*&group_id=9970) | + +`extension` and `blob` return exact matches only. + +## Examples + +| Use | Description | +|------|-------------| +| [`rails -filename:gemfile.lock`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=rails+-filename%3Agemfile.lock&snippets=) | Show _rails_ in all files except the _`gemfile.lock`_ file. | +| [`RSpec.describe Resolvers -*builder`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=RSpec.describe+Resolvers+-*builder) | Show all _RSpec.describe Resolvers_ that don't start with _builder_. | +| [<code>bug | (display +banner)</code>](https://gitlab.com/search?snippets=&scope=issues&repository_ref=&search=bug+%7C+%28display+%2Bbanner%29&group_id=9970&project_id=278964) | Show _bug_ **or** _display_ **and** _banner_. | + +<!-- markdownlint-enable --> diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 73a993c0fcf..65680227ad0 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -34,7 +34,7 @@ prompt, command shell, and command line). Here are some options: - For macOS users: - Built-in [Terminal](https://blog.teamtreehouse.com/introduction-to-the-mac-os-x-command-line). Press <kbd>⌘ command</kbd> + <kbd>space</kbd> and type `terminal`. - - [iTerm2](https://iterm2.com/). You can integrate it with [zsh](https://git-scm.com/book/id/v2/Appendix-A%3A-Git-in-Other-Environments-Git-in-Zsh) and [oh my zsh](https://ohmyz.sh/) for color highlighting and other advanced features. + - [iTerm2](https://iterm2.com/). You can integrate it with [Zsh](https://git-scm.com/book/id/v2/Appendix-A%3A-Git-in-Other-Environments-Git-in-Zsh) and [Oh My Zsh](https://ohmyz.sh/) for color highlighting and other advanced features. - For Windows users: - Built-in command line. On the Windows taskbar, select the search icon and type `cmd`. - [PowerShell](https://docs.microsoft.com/en-us/powershell/scripting/windows-powershell/install/installing-windows-powershell?view=powershell-7). @@ -149,7 +149,7 @@ between your computer and GitLab. 1. GitLab requests your username and password: - If you have 2FA enabled for your account, you must use a [Personal Access Token](../user/profile/personal_access_tokens.md) - with **read_repository** or **write_repository** permissions instead of your account's password. + with `read_repository` or `write_repository` permissions instead of your account's password. - If you don't have 2FA enabled, use your account's password. 1. To view the files, go to the new directory: diff --git a/doc/index.md b/doc/index.md index 2f3bfefe819..1ac8cdb19ce 100644 --- a/doc/index.md +++ b/doc/index.md @@ -50,7 +50,7 @@ Have a look at some of our most popular topics: | [Activate GitLab EE with a license](user/admin_area/license.md) | Activate GitLab Enterprise Edition functionality with a license. | | [Back up and restore GitLab](raketasks/backup_restore.md) | Rake tasks for backing up and restoring GitLab self-managed instances. | | [GitLab release and maintenance policy](policy/maintenance.md) | Policies for version naming and cadence, and also upgrade recommendations. | -| [Elasticsearch integration](integration/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. | +| [Elasticsearch integration](integration/advanced_search/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. | | [Omnibus GitLab database settings](https://docs.gitlab.com/omnibus/settings/database.html) | Database settings for Omnibus GitLab self-managed instances. | | [Omnibus GitLab NGINX settings](https://docs.gitlab.com/omnibus/settings/nginx.html) | NGINX settings for Omnibus GitLab self-managed instances. | | [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl.html) | SSL settings for Omnibus GitLab self-managed instances. | diff --git a/doc/install/aws/eks_clusters_aws.md b/doc/install/aws/eks_clusters_aws.md index fe436a03cb5..342740994a1 100644 --- a/doc/install/aws/eks_clusters_aws.md +++ b/doc/install/aws/eks_clusters_aws.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index 055cb4c3efb..bc811cab3bf 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -168,7 +168,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | Gitaly Instances (in ASG) | 12 vCPU, 45GB<br />(across 3 nodes) | **m5.xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.192 x 3 = $0.58/hr | $0.192 x 3 = $0.58/hr | | | The GitLab Reference architecture for 2K is not Highly Available and therefore has a single Gitaly no Praefect. AWS Quick Starts MUST be HA, so it implements Prafect from the 3K Ref Architecture to meet that requirement | | | | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 10 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | $0.09 x 3 = $0.21/hr | -| Praefect PostgreSQL(1) (AWS RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | N/A Reuses GitLab PostgreSQL | $0 | $0 | +| Praefect PostgreSQL(1) (AWS RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | $0 | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | ### 3K Cloud Native Hybrid on EKS @@ -222,7 +222,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 12 vCPU, 45GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.large** x 3 nodes<br />(12 vCPU, 48 GB) | $0.192 x 3 = $0.58/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | -| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | N/A Reuses GitLab PostgreSQL | $0 | | +| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | ### 5K Cloud Native Hybrid on EKS @@ -276,7 +276,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 24 vCPU, 90GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.2xlarge** x 3 nodes<br />(24 vCPU, 96GB) | $0.384 x 3 = $1.15/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | -| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | N/A Reuses GitLab PostgreSQL | $0 | | +| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | ### 10K Cloud Native Hybrid on EKS @@ -329,7 +329,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 48 vCPU, 180GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **m5.4xlarge** x 3 nodes<br />(48 vCPU, 180 GB) | $0.77 x 3 = $2.31/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | **c5.large** x 3 nodes<br />(6 vCPU, 12 GB) | $0.09 x 3 = $0.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | -| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | N/A Reuses GitLab PostgreSQL | $0 | | +| Praefect PostgreSQL(1) (Amazon RDS) | 6 vCPU, 5.4 GB<br />([across 3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections)) | Not applicable; reuses GitLab PostgreSQL | $0 | | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | ### 50K Cloud Native Hybrid on EKS @@ -382,7 +382,7 @@ If EKS node autoscaling is employed, it is likely that your average loading will | **<u>Gitaly Cluster</u>** [Details](gitlab_sre_for_aws.md#gitaly-sre-considerations) | | | | | | Gitaly Instances (in ASG) | 64 vCPU, 240GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | **m5.16xlarge** x 3 nodes<br />(64 vCPU, 256 GB each) | $3.07 x 3 = $9.21/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | | Praefect (Instances in ASG with load balancer) | 4 vCPU, 3.6 GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | **c5.xlarge** x 3 nodes<br />(4 vCPU, 8 GB each) | $0.17 x 3 = $0.51/hr | [Gitaly & Praefect Must Have an Uneven Node Count for HA](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | -| Praefect PostgreSQL(1) (AWS RDS) | 2 vCPU, 1.8 GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | N/A Reuses GitLab PostgreSQL | $0 | | +| Praefect PostgreSQL(1) (AWS RDS) | 2 vCPU, 1.8 GB x [3 nodes](gitlab_sre_for_aws.md#gitaly-and-praefect-elections) | Not applicable; reuses GitLab PostgreSQL | $0 | | | Internal Load Balancing Node | 2 vCPU, 1.8 GB | AWS ELB | $0.10/hr | $0.10/hr | ## Helpful Resources diff --git a/doc/install/aws/gitlab_sre_for_aws.md b/doc/install/aws/gitlab_sre_for_aws.md index 2114ed51128..cb7703835fa 100644 --- a/doc/install/aws/gitlab_sre_for_aws.md +++ b/doc/install/aws/gitlab_sre_for_aws.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index ee7279d72cd..f54bc979152 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false diff --git a/doc/install/aws/manual_install_aws.md b/doc/install/aws/manual_install_aws.md index 8b827d05b57..25973220170 100644 --- a/doc/install/aws/manual_install_aws.md +++ b/doc/install/aws/manual_install_aws.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -83,8 +83,8 @@ As we'll be using [Amazon S3 object storage](#amazon-s3-object-storage), our EC2 ### Create an IAM Policy -1. Navigate to the IAM dashboard and click on **Policies** in the left menu. -1. Click **Create policy**, select the `JSON` tab, and add a policy. We want to [follow security best practices and grant _least privilege_](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#grant-least-privilege), giving our role only the permissions needed to perform the required actions. +1. Navigate to the IAM dashboard and select **Policies** in the left menu. +1. Select **Create policy**, select the `JSON` tab, and add a policy. We want to [follow security best practices and grant _least privilege_](https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#grant-least-privilege), giving our role only the permissions needed to perform the required actions. 1. Assuming you prefix the S3 bucket names with `gl-` as shown in the diagram, add the following policy: ```json @@ -114,17 +114,17 @@ As we'll be using [Amazon S3 object storage](#amazon-s3-object-storage), our EC2 } ``` -1. Click **Review policy**, give your policy a name (we'll use `gl-s3-policy`), and click **Create policy**. +1. Select **Review policy**, give your policy a name (we'll use `gl-s3-policy`), and select **Create policy**. ### Create an IAM Role -1. Still on the IAM dashboard, click on **Roles** in the left menu, and - click **Create role**. -1. Create a new role by selecting **AWS service > EC2**, then click +1. Still on the IAM dashboard, select **Roles** in the left menu, and + select **Create role**. +1. Create a new role by selecting **AWS service > EC2**, then select **Next: Permissions**. -1. In the policy filter, search for the `gl-s3-policy` we created above, select it, and click **Tags**. -1. Add tags if needed and click **Review**. -1. Give the role a name (we'll use `GitLabS3Access`) and click **Create Role**. +1. In the policy filter, search for the `gl-s3-policy` we created above, select it, and select **Tags**. +1. Add tags if needed and select **Review**. +1. Give the role a name (we'll use `GitLabS3Access`) and select **Create Role**. We'll use this role when we [create a launch configuration](#create-a-launch-configuration) later on. @@ -140,14 +140,14 @@ Internet Gateway. We'll now create a VPC, a virtual networking environment that you'll control: 1. Sign in to [Amazon Web Services](https://console.aws.amazon.com/vpc/home). -1. Select **Your VPCs** from the left menu and then click **Create VPC**. +1. Select **Your VPCs** from the left menu and then select **Create VPC**. At the "Name tag" enter `gitlab-vpc` and at the "IPv4 CIDR block" enter `10.0.0.0/16`. If you don't require dedicated hardware, you can leave - "Tenancy" as default. Click **Yes, Create** when ready. + "Tenancy" as default. Select **Yes, Create** when ready.  -1. Select the VPC, click **Actions**, click **Edit DNS resolution**, and enable DNS resolution. Hit **Save** when done. +1. Select the VPC, select **Actions**, select **Edit DNS resolution**, and enable DNS resolution. Hit **Save** when done. ### Subnets @@ -160,7 +160,7 @@ We will create private and public subnets to match load balancers and RDS instances as well: 1. Select **Subnets** from the left menu. -1. Click **Create subnet**. Give it a descriptive name tag based on the IP, +1. Select **Create subnet**. Give it a descriptive name tag based on the IP, for example `gitlab-public-10.0.0.0`, select the VPC we created previously, select an availability zone (we'll use `us-west-2a`), and at the IPv4 CIDR block let's give it a 24 subnet `10.0.0.0/24`: @@ -176,7 +176,7 @@ RDS instances as well: | `gitlab-private-10.0.3.0` | private | `us-west-2b` | `10.0.3.0/24` | 1. Once all the subnets are created, enable **Auto-assign IPv4** for the two public subnets: - 1. Select each public subnet in turn, click **Actions**, and click **Modify auto-assign IP settings**. Enable the option and save. + 1. Select each public subnet in turn, select **Actions**, and select **Modify auto-assign IP settings**. Enable the option and save. ### Internet Gateway @@ -184,8 +184,8 @@ Now, still on the same dashboard, go to Internet Gateways and create a new one: 1. Select **Internet Gateways** from the left menu. -1. Click **Create internet gateway**, give it the name `gitlab-gateway` and - click **Create**. +1. Select **Create internet gateway**, give it the name `gitlab-gateway` and + select **Create**. 1. Select it from the table, and then under the **Actions** dropdown choose "Attach to VPC". @@ -197,12 +197,12 @@ create a new one: Instances deployed in our private subnets need to connect to the internet for updates, but should not be reachable from the public internet. To achieve this, we'll make use of [NAT Gateways](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html) deployed in each of our public subnets: -1. Navigate to the VPC dashboard and click on **NAT Gateways** in the left menu bar. -1. Click **Create NAT Gateway** and complete the following: +1. Navigate to the VPC dashboard and select **NAT Gateways** in the left menu bar. +1. Select **Create NAT Gateway** and complete the following: 1. **Subnet**: Select `gitlab-public-10.0.0.0` from the dropdown. - 1. **Elastic IP Allocation ID**: Enter an existing Elastic IP or click **Allocate Elastic IP address** to allocate a new IP to your NAT gateway. + 1. **Elastic IP Allocation ID**: Enter an existing Elastic IP or select **Allocate Elastic IP address** to allocate a new IP to your NAT gateway. 1. Add tags if needed. - 1. Click **Create NAT Gateway**. + 1. Select **Create NAT Gateway**. Create a second NAT gateway but this time place it in the second public subnet, `gitlab-public-10.0.2.0`. @@ -215,23 +215,23 @@ We need to create a route table for our public subnets to reach the internet via On the VPC dashboard: 1. Select **Route Tables** from the left menu. -1. Click **Create Route Table**. +1. Select **Create Route Table**. 1. At the "Name tag" enter `gitlab-public` and choose `gitlab-vpc` under "VPC". -1. Click **Create**. +1. Select **Create**. We now need to add our internet gateway as a new target and have it receive traffic from any destination. 1. Select **Route Tables** from the left menu and select the `gitlab-public` route to show the options at the bottom. -1. Select the **Routes** tab, click **Edit routes > Add route** and set `0.0.0.0/0` +1. Select the **Routes** tab, select **Edit routes > Add route** and set `0.0.0.0/0` as the destination. In the target column, select the `gitlab-gateway` we created previously. Hit **Save routes** once done. Next, we must associate the **public** subnets to the route table: -1. Select the **Subnet Associations** tab and click **Edit subnet associations**. -1. Check only the public subnets and click **Save**. +1. Select the **Subnet Associations** tab and select **Edit subnet associations**. +1. Check only the public subnets and select **Save**. #### Private Route Tables @@ -251,7 +251,7 @@ We'll create a load balancer to evenly distribute inbound traffic on ports `80` On the EC2 dashboard, look for Load Balancer in the left navigation bar: -1. Click the **Create Load Balancer** button. +1. Select **Create Load Balancer**. 1. Choose the **Classic Load Balancer**. 1. Give it a name (we'll use `gitlab-loadbalancer`) and for the **Create LB Inside** option, select `gitlab-vpc` from the dropdown menu. 1. In the **Listeners** section, set the following listeners: @@ -259,20 +259,20 @@ On the EC2 dashboard, look for Load Balancer in the left navigation bar: - TCP port 22 for both load balancer and instance protocols and ports - HTTPS port 443 for load balancer protocol and ports, forwarding to HTTP port 80 on the instance (we will configure GitLab to listen on port 80 [later in the guide](#add-support-for-proxied-ssl)) 1. In the **Select Subnets** section, select both public subnets from the list so that the load balancer can route traffic to both availability zones. -1. We'll add a security group for our load balancer to act as a firewall to control what traffic is allowed through. Click **Assign Security Groups** and select **Create a new security group**, give it a name +1. We'll add a security group for our load balancer to act as a firewall to control what traffic is allowed through. Select **Assign Security Groups** and select **Create a new security group**, give it a name (we'll use `gitlab-loadbalancer-sec-group`) and description, and allow both HTTP and HTTPS traffic from anywhere (`0.0.0.0/0, ::/0`). Also allow SSH traffic, select a custom source, and add a single trusted IP address or an IP address range in CIDR notation. This will allow users to perform Git actions over SSH. -1. Click **Configure Security Settings** and set the following: +1. Select **Configure Security Settings** and set the following: 1. Select an SSL/TLS certificate from ACM or upload a certificate to IAM. 1. Under **Select a Cipher**, pick a predefined security policy from the dropdown. You can see a breakdown of [Predefined SSL Security Policies for Classic Load Balancers](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-security-policy-table.html) in the AWS docs. Check the GitLab codebase for a list of [supported SSL ciphers and protocols](https://gitlab.com/gitlab-org/gitlab/-/blob/9ee7ad433269b37251e0dd5b5e00a0f00d8126b4/lib/support/nginx/gitlab-ssl#L97-99). -1. Click **Configure Health Check** and set up a health check for your EC2 instances. +1. Select **Configure Health Check** and set up a health check for your EC2 instances. 1. For **Ping Protocol**, select HTTP. 1. For **Ping Port**, enter 80. - 1. For **Ping Path** - we recommend that you [use the Readiness check endpoint](../../administration/load_balancer.md#readiness-check). You'll need to add [the VPC IP Address Range (CIDR)](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-security-groups.html#elb-vpc-nacl) to the [IP Allowlist](../../administration/monitoring/ip_whitelist.md) for the [Health Check endpoints](../../user/admin_area/monitoring/health_check.md) + 1. For **Ping Path** - we recommend that you [use the Readiness check endpoint](../../administration/load_balancer.md#readiness-check). You'll need to add [the VPC IP Address Range (CIDR)](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-security-groups.html#elb-vpc-nacl) to the [IP allowlist](../../administration/monitoring/ip_whitelist.md) for the [Health Check endpoints](../../user/admin_area/monitoring/health_check.md) 1. Keep the default **Advanced Details** or adjust them according to your needs. -1. Click **Add EC2 Instances** - don't add anything as we will create an Auto Scaling Group later to manage instances for us. -1. Click **Add Tags** and add any tags you need. -1. Click **Review and Create**, review all your settings, and click **Create** if you're happy. +1. Select **Add EC2 Instances** - don't add anything as we will create an Auto Scaling Group later to manage instances for us. +1. Select **Add Tags** and add any tags you need. +1. Select **Review and Create**, review all your settings, and select **Create** if you're happy. After the Load Balancer is up and running, you can revisit your Security Groups to refine the access only through the ELB and any other requirements @@ -280,19 +280,19 @@ you might have. ### Configure DNS for Load Balancer -On the Route 53 dashboard, click **Hosted zones** in the left navigation bar: +On the Route 53 dashboard, select **Hosted zones** in the left navigation bar: -1. Select an existing hosted zone or, if you do not already have one for your domain, click **Create Hosted Zone**, enter your domain name, and click **Create**. -1. Click **Create Record Set** and provide the following values: +1. Select an existing hosted zone or, if you do not already have one for your domain, select **Create Hosted Zone**, enter your domain name, and select **Create**. +1. Select **Create Record Set** and provide the following values: 1. **Name:** Use the domain name (the default value) or enter a subdomain. 1. **Type:** Select **A - IPv4 address**. 1. **Alias:** Defaults to **No**. Select **Yes**. 1. **Alias Target:** Find the **ELB Classic Load Balancers** section and select the classic load balancer we created earlier. 1. **Routing Policy:** We'll use **Simple** but you can choose a different policy based on your use case. 1. **Evaluate Target Health:** We'll set this to **No** but you can choose to have the load balancer route traffic based on target health. - 1. Click **Create**. + 1. Select **Create**. 1. If you registered your domain through Route 53, you're done. If you used a different domain registrar, you need to update your DNS records with your domain registrar. You'll need to: - 1. Click on **Hosted zones** and select the domain you added above. + 1. Select **Hosted zones** and select the domain you added above. 1. You'll see a list of `NS` records. From your domain registrar's administrator panel, add each of these as `NS` records to your domain's DNS records. These steps may vary between domain registrars. If you're stuck, Google **"name of your registrar" add DNS records** and you should find a help article specific to your domain registrar. The steps for doing this vary depending on which registrar you use and is beyond the scope of this guide. @@ -308,22 +308,22 @@ create the actual RDS instance. We need a security group for our database that will allow inbound traffic from the instances we'll deploy in our `gitlab-loadbalancer-sec-group` later on: 1. From the EC2 dashboard, select **Security Groups** from the left menu bar. -1. Click **Create security group**. +1. Select **Create security group**. 1. Give it a name (we'll use `gitlab-rds-sec-group`), a description, and select the `gitlab-vpc` from the **VPC** dropdown. -1. In the **Inbound rules** section, click **Add rule** and set the following: +1. In the **Inbound rules** section, select **Add rule** and set the following: 1. **Type:** search for and select the **PostgreSQL** rule. 1. **Source type:** set as "Custom". 1. **Source:** select the `gitlab-loadbalancer-sec-group` we created earlier. -1. When done, click **Create security group**. +1. When done, select **Create security group**. ### RDS Subnet Group 1. Navigate to the RDS dashboard and select **Subnet Groups** from the left menu. -1. Click on **Create DB Subnet Group**. +1. Select **Create DB Subnet Group**. 1. Under **Subnet group details**, enter a name (we'll use `gitlab-rds-group`), a description, and choose the `gitlab-vpc` from the VPC dropdown. 1. From the **Availability Zones** dropdown, select the Availability Zones that include the subnets you've configured. In our case, we'll add `eu-west-2a` and `eu-west-2b`. 1. From the **Subnets** dropdown, select the two private subnets (`10.0.1.0/24` and `10.0.3.0/24`) as we defined them in the [subnets section](#subnets). -1. Click **Create** when ready. +1. Select **Create** when ready. ### Create the database @@ -332,7 +332,7 @@ Avoid using burstable instances (t class instances) for the database as this cou Now, it's time to create the database: -1. Navigate to the RDS dashboard, select **Databases** from the left menu, and click **Create database**. +1. Navigate to the RDS dashboard, select **Databases** from the left menu, and select **Create database**. 1. Select **Standard Create** for the database creation method. 1. Select **PostgreSQL** as the database engine and select the minimum PostgreSQL version as defined for your GitLab version in our [database requirements](../../install/requirements.md#postgresql-requirements). 1. Since this is a production server, let's choose **Production** from the **Templates** section. @@ -355,7 +355,7 @@ Now, it's time to create the database: 1. Configure your preferred backup settings. 1. The only other change we'll make here is to disable auto minor version updates under **Maintenance**. 1. Leave all the other settings as is or tweak according to your needs. - 1. Once you're happy, click **Create database**. + 1. Once you're happy, select **Create database**. Now that the database is created, let's move on to setting up Redis with ElastiCache. @@ -368,24 +368,24 @@ persistence and is used to store session data, temporary cache information, and 1. Navigate to the EC2 dashboard. 1. Select **Security Groups** from the left menu. -1. Click **Create security group** and fill in the details. Give it a name (we'll use `gitlab-redis-sec-group`), +1. Select **Create security group** and fill in the details. Give it a name (we'll use `gitlab-redis-sec-group`), add a description, and choose the VPC we created previously -1. In the **Inbound rules** section, click **Add rule** and add a **Custom TCP** rule, set port `6379`, and set the "Custom" source as the `gitlab-loadbalancer-sec-group` we created earlier. -1. When done, click **Create security group**. +1. In the **Inbound rules** section, select **Add rule** and add a **Custom TCP** rule, set port `6379`, and set the "Custom" source as the `gitlab-loadbalancer-sec-group` we created earlier. +1. When done, select **Create security group**. ### Redis Subnet Group 1. Navigate to the ElastiCache dashboard from your AWS console. 1. Go to **Subnet Groups** in the left menu, and create a new subnet group (we'll name ours `gitlab-redis-group`). - Make sure to select our VPC and its [private subnets](#subnets). Click - **Create** when ready. + Make sure to select our VPC and its [private subnets](#subnets). +1. Select **Create** when ready.  ### Create the Redis Cluster 1. Navigate back to the ElastiCache dashboard. -1. Select **Redis** on the left menu and click **Create** to create a new +1. Select **Redis** on the left menu and select **Create** to create a new Redis cluster. Do not enable **Cluster Mode** as it is [not supported](../../administration/redis/replication_and_failover_external.md#requirements). Even without cluster mode on, you still get the chance to deploy Redis in multiple availability zones. 1. In the settings section: @@ -404,7 +404,7 @@ persistence and is used to store session data, temporary cache information, and 1. In the security settings, edit the security groups and choose the `gitlab-redis-sec-group` we had previously created. 1. Leave the rest of the settings to their default values or edit to your liking. -1. When done, click **Create**. +1. When done, select **Create**. ## Setting up Bastion Hosts @@ -418,29 +418,29 @@ If you do not want to maintain bastion hosts, you can set up [AWS Systems Manage ### Create Bastion Host A -1. Navigate to the EC2 Dashboard and click on **Launch instance**. +1. Navigate to the EC2 Dashboard and select **Launch instance**. 1. Select the **Ubuntu Server 18.04 LTS (HVM)** AMI. 1. Choose an instance type. We'll use a `t2.micro` as we'll only use the bastion host to SSH into our other instances. -1. Click **Configure Instance Details**. +1. Select **Configure Instance Details**. 1. Under **Network**, select the `gitlab-vpc` from the dropdown menu. 1. Under **Subnet**, select the public subnet we created earlier (`gitlab-public-10.0.0.0`). 1. Double check that under **Auto-assign Public IP** you have **Use subnet setting (Enable)** selected. - 1. Leave everything else as default and click **Add Storage**. + 1. Leave everything else as default and select **Add Storage**. 1. For storage, we'll leave everything as default and only add an 8GB root volume. We won't store anything on this instance. -1. Click **Add Tags** and on the next screen click **Add Tag**. +1. Select **Add Tags** and on the next screen select **Add Tag**. 1. We'll only set `Key: Name` and `Value: Bastion Host A`. -1. Click **Configure Security Group**. +1. Select **Configure Security Group**. 1. Select **Create a new security group**, enter a **Security group name** (we'll use `bastion-sec-group`), and add a description. 1. We'll enable SSH access from anywhere (`0.0.0.0/0`). If you want stricter security, specify a single IP address or an IP address range in CIDR notation. - 1. Click **Review and Launch** -1. Review all your settings and, if you're happy, click **Launch**. -1. Acknowledge that you have access to an existing key pair or create a new one. Click **Launch Instance**. + 1. Select **Review and Launch** +1. Review all your settings and, if you're happy, select **Launch**. +1. Acknowledge that you have access to an existing key pair or create a new one. Select **Launch Instance**. Confirm that you can SSH into the instance: -1. On the EC2 Dashboard, click on **Instances** in the left menu. +1. On the EC2 Dashboard, select **Instances** in the left menu. 1. Select **Bastion Host A** from your list of instances. -1. Click **Connect** and follow the connection instructions. +1. Select **Connect** and follow the connection instructions. 1. If you are able to connect successfully, let's move on to setting up our second bastion host for redundancy. ### Create Bastion Host B @@ -466,16 +466,16 @@ From the EC2 dashboard: 1. Use the section below titled "[Find official GitLab-created AMI IDs on AWS](#find-official-gitlab-created-ami-ids-on-aws)" to find the correct AMI to launch. 1. After clicking **Launch** on the desired AMI, select an instance type based on your workload. Consult the [hardware requirements](../../install/requirements.md#hardware-requirements) to choose one that fits your needs (at least `c5.xlarge`, which is sufficient to accommodate 100 users). -1. Click **Configure Instance Details**: +1. Select **Configure Instance Details**: 1. In the **Network** dropdown, select `gitlab-vpc`, the VPC we created earlier. 1. In the **Subnet** dropdown, select `gitlab-private-10.0.1.0` from the list of subnets we created earlier. 1. Double check that **Auto-assign Public IP** is set to `Use subnet setting (Disable)`. - 1. Click **Add Storage**. + 1. Select **Add Storage**. 1. The root volume is 8GiB by default and should be enough given that we won't store any data there. -1. Click **Add Tags** and add any tags you may need. In our case, we'll only set `Key: Name` and `Value: GitLab`. -1. Click **Configure Security Group**. Check **Select an existing security group** and select the `gitlab-loadbalancer-sec-group` we created earlier. -1. Click **Review and launch** followed by **Launch** if you're happy with your settings. -1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. +1. Select **Add Tags** and add any tags you may need. In our case, we'll only set `Key: Name` and `Value: GitLab`. +1. Select **Configure Security Group**. Check **Select an existing security group** and select the `gitlab-loadbalancer-sec-group` we created earlier. +1. Select **Review and launch** followed by **Launch** if you're happy with your settings. +1. Finally, acknowledge that you have access to the selected private key file or create a new one. Select **Launch Instances**. ### Add custom configuration @@ -501,7 +501,7 @@ Since we're adding our SSL certificate at the load balancer, we do not need the From your GitLab instance, connect to the RDS instance to verify access and to install the required `pg_trgm` and `btree_gist` extensions. -To find the host or endpoint, navigate to **Amazon RDS > Databases** and click on the database you created earlier. Look for the endpoint under the **Connectivity & security** tab. +To find the host or endpoint, navigate to **Amazon RDS > Databases** and select the database you created earlier. Look for the endpoint under the **Connectivity & security** tab. Do not to include the colon and port number: @@ -580,23 +580,23 @@ It should be enabled and configured on a separate EC2 instance in one of the Let's create an EC2 instance where we'll install Gitaly: -1. From the EC2 dashboard, click **Launch instance**. +1. From the EC2 dashboard, select **Launch instance**. 1. Choose an AMI. In this example, we'll select the **Ubuntu Server 18.04 LTS (HVM), SSD Volume Type**. 1. Choose an instance type. We'll pick a `c5.xlarge`. -1. Click **Configure Instance Details**. +1. Select **Configure Instance Details**. 1. In the **Network** dropdown, select `gitlab-vpc`, the VPC we created earlier. 1. In the **Subnet** dropdown, select `gitlab-private-10.0.1.0` from the list of subnets we created earlier. 1. Double check that **Auto-assign Public IP** is set to `Use subnet setting (Disable)`. - 1. Click **Add Storage**. + 1. Select **Add Storage**. 1. Increase the Root volume size to `20 GiB` and change the **Volume Type** to `Provisioned IOPS SSD (io1)`. (This is an arbitrary size. Create a volume big enough for your repository storage requirements.) 1. For **IOPS** set `1000` (20 GiB x 50 IOPS). You can provision up to 50 IOPS per GiB. If you select a larger volume, increase the IOPS accordingly. Workloads where many small files are written in a serialized manner, like `git`, requires performant storage, hence the choice of `Provisioned IOPS SSD (io1)`. -1. Click on **Add Tags** and add your tags. In our case, we'll only set `Key: Name` and `Value: Gitaly`. -1. Click on **Configure Security Group** and let's **Create a new security group**. +1. Select **Add Tags** and add your tags. In our case, we'll only set `Key: Name` and `Value: Gitaly`. +1. Select **Configure Security Group** and let's **Create a new security group**. 1. Give your security group a name and description. We'll use `gitlab-gitaly-sec-group` for both. 1. Create a **Custom TCP** rule and add port `8075` to the **Port Range**. For the **Source**, select the `gitlab-loadbalancer-sec-group`. 1. Also add an inbound rule for SSH from the `bastion-sec-group` so that we can connect using [SSH Agent Forwarding](#use-ssh-agent-forwarding) from the Bastion hosts. -1. Click **Review and launch** followed by **Launch** if you're happy with your settings. -1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. +1. Select **Review and launch** followed by **Launch** if you're happy with your settings. +1. Finally, acknowledge that you have access to the selected private key file or create a new one. Select **Launch Instances**. NOTE: Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). We do not recommend using EFS as it may negatively impact the performance of GitLab. You can review the [relevant documentation](../../administration/nfs.md#avoid-using-cloud-based-file-systems) for more details. @@ -679,9 +679,9 @@ When our [auto scaling group](#create-an-auto-scaling-group) spins up new instan On the EC2 dashboard: 1. Select the `GitLab` instance we [created earlier](#install-gitlab). -1. Click on **Actions**, scroll down to **Image** and click **Create Image**. +1. Select **Actions**, scroll down to **Image** and select **Create Image**. 1. Give your image a name and description (we'll use `GitLab-Source` for both). -1. Leave everything else as default and click **Create Image** +1. Leave everything else as default and select **Create Image** Now we have a custom AMI that we'll use to create our launch configuration the next step. @@ -691,21 +691,21 @@ Now we have a custom AMI that we'll use to create our launch configuration the n From the EC2 dashboard: -1. Select **Launch Configurations** from the left menu and click **Create launch configuration**. +1. Select **Launch Configurations** from the left menu and select **Create launch configuration**. 1. Select **My AMIs** from the left menu and select the `GitLab` custom AMI we created above. -1. Select an instance type best suited for your needs (at least a `c5.xlarge`) and click **Configure details**. +1. Select an instance type best suited for your needs (at least a `c5.xlarge`) and select **Configure details**. 1. Enter a name for your launch configuration (we'll use `gitlab-ha-launch-config`). 1. **Do not** check **Request Spot Instance**. 1. From the **IAM Role** dropdown, pick the `GitLabAdmin` instance role we [created earlier](#create-an-iam-ec2-instance-role-and-profile). -1. Leave the rest as defaults and click **Add Storage**. -1. The root volume is 8GiB by default and should be enough given that we won't store any data there. Click **Configure Security Group**. +1. Leave the rest as defaults and select **Add Storage**. +1. The root volume is 8GiB by default and should be enough given that we won't store any data there. Select **Configure Security Group**. 1. Check **Select and existing security group** and select the `gitlab-loadbalancer-sec-group` we created earlier. -1. Click **Review**, review your changes, and click **Create launch configuration**. -1. Acknowledge that you have access to the private key or create a new one. Click **Create launch configuration**. +1. Select **Review**, review your changes, and select **Create launch configuration**. +1. Acknowledge that you have access to the private key or create a new one. Select **Create launch configuration**. ### Create an auto scaling group -1. As soon as the launch configuration is created, you'll see an option to **Create an Auto Scaling group using this launch configuration**. Click that to start creating the auto scaling group. +1. After the launch configuration is created, select **Create an Auto Scaling group using this launch configuration** to start creating the auto scaling group. 1. Enter a **Group name** (we'll use `gitlab-auto-scaling-group`). 1. For **Group size**, enter the number of instances you want to start with (we'll enter `2`). 1. Select the `gitlab-vpc` from the **Network** dropdown. @@ -713,7 +713,7 @@ From the EC2 dashboard: 1. Expand the **Advanced Details** section and check the **Receive traffic from one or more load balancers** option. 1. From the **Classic Load Balancers** dropdown, select the load balancer we created earlier. 1. For **Health Check Type**, select **ELB**. -1. We'll leave our **Health Check Grace Period** as the default `300` seconds. Click **Configure scaling policies**. +1. We'll leave our **Health Check Grace Period** as the default `300` seconds. Select **Configure scaling policies**. 1. Check **Use scaling policies to adjust the capacity of this group**. 1. For this group we'll scale between 2 and 4 instances where one instance will be added if CPU utilization is greater than 60% and one instance is removed if it falls diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index 780dfe17dac..0d621217dd8 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments description: 'Learn how to spin up a pre-configured GitLab VM on Microsoft Azure.' @@ -15,7 +15,7 @@ Enterprise Edition in a single Virtual Machine (VM). ## Prerequisite -You'll need an account on Azure. Use of the following methods to obtain an account: +You need an account on Azure. Use of the following methods to obtain an account: - If you or your company already have an account with a subscription, use that account. If not, you can [open your own Azure account for free](https://azure.microsoft.com/en-us/free/). @@ -155,7 +155,7 @@ to assign a descriptive DNS name to the VM: 1. Enter a descriptive DNS name for your instance in the **DNS name label** field, for example `gitlab-prod`. This makes the VM accessible at `gitlab-prod.eastus.cloudapp.azure.com`. -1. Select **Save** for the changes to take effect. +1. Select **Save**. Eventually, most users want to use their own domain name. For you to do this, you need to add a DNS `A` record with your domain registrar that points to the public IP address of your Azure VM. diff --git a/doc/install/cloud_native/index.md b/doc/install/cloud_native/index.md index 45d484b045a..f3265500877 100644 --- a/doc/install/cloud_native/index.md +++ b/doc/install/cloud_native/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false diff --git a/doc/install/digitaloceandocker.md b/doc/install/digitaloceandocker.md index c0b9b280d92..86ccf194786 100644 --- a/doc/install/digitaloceandocker.md +++ b/doc/install/digitaloceandocker.md @@ -1,150 +1,11 @@ --- -stage: Enablement -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'docker.md' +remove_date: '2022-08-29' --- -# Digital Ocean and Docker Machine test environment **(FREE SELF)** +This document was moved to [another location](docker.md). -This guide is for quickly testing different versions of GitLab and not -recommended for ease of future upgrades or keeping the data you create. - -## Initial setup - -This guide configures a Digital Ocean droplet and sets up Docker -locally on either macOS or Linux. - -### On macOS - -#### Install Docker Desktop - -- <https://www.docker.com/products/docker-desktop> - -### On Linux - -#### Install Docker Engine - -- <https://docs.docker.com/engine/installation/linux/> - -#### Install Docker Machine - -- <https://docs.docker.com/machine/install-machine/> - -NOTE: -The rest of the steps are identical for macOS and Linux. - -## Create new Docker host - -1. Login to Digital Ocean. -1. Generate a new API token at <https://cloud.digitalocean.com/settings/api/tokens>. - - This command creates a new Digital Ocean droplet called `gitlab-test-env-do` that acts as a Docker host. - - NOTE: - 4GB is the minimum requirement for a Docker host that runs more than one GitLab instance. - - - RAM: 4GB - - Name: `gitlab-test-env-do` - - Driver: `digitalocean` - -1. Set the DO token: - - ```shell - export DOTOKEN=<your generated token> - ``` - -1. Create the machine: - - ```shell - docker-machine create \ - --driver digitalocean \ - --digitalocean-access-token=$DOTOKEN \ - --digitalocean-size "4gb" \ - gitlab-test-env-do - ``` - -Resource: <https://docs.docker.com/machine/drivers/digital-ocean/>. - -## Creating GitLab test instance - -### Connect your shell to the new machine - -This example creates a GitLab EE 8.10.8 instance. - -First connect the Docker client to the Docker host you created previously. - -```shell -eval "$(docker-machine env gitlab-test-env-do)" -``` - -You can add this to your `~/.bash_profile` file to ensure the `docker` client uses the `gitlab-test-env-do` Docker host - -### Create new GitLab container - -- HTTP port: `8888` -- SSH port: `2222` - - Set `gitlab_shell_ssh_port` using `--env GITLAB_OMNIBUS_CONFIG` -- Hostname: IP of Docker host -- Container name: `gitlab-test-8.10` -- GitLab version: **EE** `8.10.8-ee.0` - -#### Set up container settings - -```shell -export SSH_PORT=2222 -export HTTP_PORT=8888 -export VERSION=8.10.8-ee.0 -export NAME=gitlab-test-8.10 -``` - -#### Create container - -```shell -docker run --detach \ ---env GITLAB_OMNIBUS_CONFIG="external_url 'http://$(docker-machine ip gitlab-test-env-do):$HTTP_PORT'; gitlab_rails['gitlab_shell_ssh_port'] = $SSH_PORT;" \ ---hostname $(docker-machine ip gitlab-test-env-do) \ --p $HTTP_PORT:$HTTP_PORT -p $SSH_PORT:22 \ ---name $NAME \ -gitlab/gitlab-ee:$VERSION -``` - -### Connect to the GitLab container - -#### Retrieve the Docker host IP - -```shell -docker-machine ip gitlab-test-env-do -# example output: 192.168.151.134 -``` - -Browse to: `http://192.168.151.134:8888/`. - -#### Execute interactive shell/edit configuration - -```shell -docker exec -it $NAME /bin/bash -``` - -```shell -# example commands -root@192:/# vi /etc/gitlab/gitlab.rb -root@192:/# gitlab-ctl reconfigure -``` - -### Resources - -- <https://docs.gitlab.com/omnibus/docker/>. -- <https://docs.docker.com/machine/get-started/>. -- <https://docs.docker.com/machine/reference/ip/>. - -<!-- ## Troubleshooting - -Include any troubleshooting steps that you can foresee. If you know beforehand what issues -one might have when setting this up, or when something is changed, or on upgrading, it's -important to describe those, too. Think of things that may go wrong and include them here. -This is important to minimize requests for support, and to avoid doc comments with -questions that you know someone might ask. - -Each scenario can be a third-level heading, e.g. `### Getting error message X`. -If you have none to add when creating a doc, leave this section in place -but commented out to help encourage others to add to it in the future. --> +<!-- This redirect file can be deleted after <2022-08-29>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/install/docker.md b/doc/install/docker.md index d5ca2c2e29f..058233520ca 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -449,14 +449,14 @@ web browser under `<hostIP>:8929` and push using SSH under the port `2289`. A `docker-compose.yml` example that uses different ports can be found in the [Docker compose](#install-gitlab-using-docker-compose) section. -## Update +## Upgrade -In most cases, updating GitLab is as easy as downloading the newest Docker +In most cases, upgrading GitLab is as easy as downloading the newest Docker [image tag](#use-tagged-versions-of-gitlab). -### Update GitLab using Docker Engine +### Upgrade GitLab using Docker Engine -To update GitLab that was [installed using Docker Engine](#install-gitlab-using-docker-engine): +To upgrade GitLab that was [installed using Docker Engine](#install-gitlab-using-docker-engine): 1. Take a [backup](#back-up-gitlab). 1. Stop the running container: @@ -493,17 +493,17 @@ To update GitLab that was [installed using Docker Engine](#install-gitlab-using- gitlab/gitlab-ee:latest ``` -On the first run, GitLab will reconfigure and update itself. +On the first run, GitLab will reconfigure and upgrade itself. -Refer to the GitLab [Update recommendations](../policy/maintenance.md#upgrade-recommendations) +Refer to the GitLab [Upgrade recommendations](../policy/maintenance.md#upgrade-recommendations) when upgrading between major versions. -### Update GitLab using Docker compose +### Upgrade GitLab using Docker compose -To update GitLab that was [installed using Docker Compose](#install-gitlab-using-docker-compose): +To upgrade GitLab that was [installed using Docker Compose](#install-gitlab-using-docker-compose): 1. Take a [backup](#back-up-gitlab). -1. Download the newest release and update your GitLab instance: +1. Download the newest release and upgrade your GitLab instance: ```shell docker-compose pull @@ -517,7 +517,7 @@ To update GitLab that was [installed using Docker Compose](#install-gitlab-using You can convert an existing Docker-based GitLab Community Edition (CE) container to a GitLab [Enterprise Edition](https://about.gitlab.com/pricing/) (EE) container -using the same approach as [updating the version](#update). +using the same approach as [upgrading the version](#upgrade). We recommend you convert from the same version of CE to EE (for example, CE 14.1 to EE 14.1). This is not explicitly necessary, and any standard upgrade (for example, CE 14.0 to EE 14.1) should work. @@ -643,7 +643,7 @@ sudo setfacl -mR default:group:docker:rwx $GITLAB_HOME The default group is `docker`. If you changed the group, be sure to update your commands. -### /dev/shm mount not having enough space in Docker container +### `/dev/shm` mount not having enough space in Docker container GitLab comes with a Prometheus metrics endpoint at `/-/metrics` to expose a variety of statistics on the health and performance of GitLab. The files diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 76c1da74108..b63ab8d4e2b 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments description: 'Learn how to install a GitLab instance on Google Cloud Platform.' @@ -13,7 +13,7 @@ NOTE: To deploy production-ready GitLab on Google Kubernetes Engine, you can follow Google Cloud Platform's -[Click to Deploy steps](https://github.com/GoogleCloudPlatform/click-to-deploy/blob/master/k8s/gitlab/README.md) +[`Click to Deploy` steps](https://github.com/GoogleCloudPlatform/click-to-deploy/blob/master/k8s/gitlab/README.md) It's an alternative to using a GCP VM, and uses the [Cloud native GitLab Helm chart](https://docs.gitlab.com/charts/). diff --git a/doc/install/index.md b/doc/install/index.md index 52bc9062adc..413c5116c14 100644 --- a/doc/install/index.md +++ b/doc/install/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -45,7 +45,6 @@ methods, the majority which use the Linux packages: | [AWS (HA)](aws/index.md) | Install GitLab on AWS using the community AMIs provided by GitLab. | | [Google Cloud Platform (GCP)](google_cloud_platform/index.md) | Install GitLab on a VM in GCP. | | [Azure](azure/index.md) | Install GitLab from Azure Marketplace. | -| [DigitalOcean](https://about.gitlab.com/blog/2016/04/27/getting-started-with-gitlab-and-digitalocean/) | Install GitLab on DigitalOcean. You can also [test GitLab on DigitalOcean using Docker Machine](digitaloceandocker.md). | ## Next steps diff --git a/doc/install/installation.md b/doc/install/installation.md index e53ced6c88a..cc2e57aac96 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -117,7 +117,7 @@ Install the required packages (needed to compile Ruby and native extensions to R ```shell sudo apt-get install -y build-essential zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libre2-dev \ libreadline-dev libncurses5-dev libffi-dev curl openssh-server libxml2-dev libxslt-dev \ - libcurl4-openssl-dev libicu-dev logrotate rsync python-docutils pkg-config cmake runit-systemd + libcurl4-openssl-dev libicu-dev logrotate rsync python3-docutils pkg-config cmake runit-systemd ``` If you want to use Kerberos for user authentication, install `libkrb5-dev` @@ -211,7 +211,7 @@ sudo apt-get install -y libimage-exiftool-perl ## 2. Ruby The Ruby interpreter is required to run GitLab. -See the [requirements section of this page](#software-requirements) for the minimum +See the [requirements section](#software-requirements) for the minimum Ruby requirements. The use of Ruby version managers such as [`RVM`](https://rvm.io/), [`rbenv`](https://github.com/rbenv/rbenv) or [`chruby`](https://github.com/postmodern/chruby) with GitLab @@ -226,9 +226,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz" -echo '2755b900a21235b443bb16dadd9032f784d4a88f143d852bc5d154f22b8781f1 ruby-2.7.5.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.5.tar.gz -cd ruby-2.7.5 +curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.6.tar.gz" +echo 'e7203b0cc09442ed2c08936d483f8ac140ec1c72e37bb5c401646b7866cb5d10 ruby-2.7.6.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.6.tar.gz +cd ruby-2.7.6 ./configure --disable-install-rdoc --enable-shared make @@ -246,11 +246,11 @@ page](https://go.dev/dl). # Remove former Go installation folder sudo rm -rf /usr/local/go -curl --remote-name --location --progress-bar "https://go.dev/dl/go1.16.10.linux-amd64.tar.gz" -echo '414cd18ce1d193769b9e97d2401ad718755ab47816e13b2a1cde203d263b55cf go1.16.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ - sudo tar -C /usr/local -xzf go1.16.10.linux-amd64.tar.gz +curl --remote-name --location --progress-bar "https://go.dev/dl/go1.17.10.linux-amd64.tar.gz" +echo '87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd go1.17.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.17.10.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,gofmt} /usr/local/bin/ -rm go1.16.10.linux-amd64.tar.gz +rm go1.17.10.linux-amd64.tar.gz ``` ## 4. Node @@ -859,7 +859,7 @@ You can set the Administrator/root password and email by supplying them in envir sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production GITLAB_ROOT_PASSWORD=yourpassword GITLAB_ROOT_EMAIL=youremail GITLAB_LICENSE_FILE="/path/to/license" ``` -### Secure secrets.yml +### Secure `secrets.yml` The `secrets.yml` file stores encryption keys for sessions and secure variables. Backup `secrets.yml` someplace safe, but don't store it in the same place as your database backups. diff --git a/doc/install/next_steps.md b/doc/install/next_steps.md index 5d8c61cad44..4defb62d254 100644 --- a/doc/install/next_steps.md +++ b/doc/install/next_steps.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -56,7 +56,7 @@ installation. ## Cross-repository Code Search -- [Advanced Search](../integration/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for +- [Advanced Search](../integration/advanced_search/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for faster, more advanced code search across your entire GitLab instance. ## Scaling and replication diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index 364c27f089f..6bce71e84c1 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index 831e33870bd..6c60ee09d78 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 83fce2f00f6..d01860cb24f 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -13,15 +13,7 @@ the minimum requirements needed to install and use GitLab. ### Supported Linux distributions -- Ubuntu (18.04/20.04) -- Debian (9/10/11) -- AlmaLinux (8) -- CentOS (7) -- openSUSE Leap (15.3) -- SUSE Linux Enterprise Server (12 SP2/12 SP5) -- Red Hat Enterprise Linux (use the AlmaLinux or CentOS instructions) -- Scientific Linux (use the CentOS instructions) -- Oracle Linux (use the CentOS instructions) +See the [list of supported operating systems](../administration/package_information/supported_os.md#supported-operating-systems). For the installation options, see [the main installation page](index.md). @@ -121,8 +113,8 @@ the following table) as these were used for development and testing: |----------------|----------------------------| | 13.0 | 11 | | 14.0 | 12.10 | -| 15.0 | 12.0 | -| 16.0 (planned) | 13.0 | +| 15.0 | 12.10 | +| 16.0 (planned) | 13.6 | You must also ensure the following extensions are loaded into every GitLab database. [Read more about this requirement, and troubleshooting](postgresql_extensions.md). diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md new file mode 100644 index 00000000000..c123611fa44 --- /dev/null +++ b/doc/integration/advanced_search/elasticsearch.md @@ -0,0 +1,857 @@ +--- +type: reference +stage: Data Stores +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Elasticsearch integration **(PREMIUM SELF)** + +This page describes how to enable Advanced Search. When enabled, +Advanced Search provides faster search response times and [improved search features](../../user/search/advanced_search.md). + +## Version requirements + +### Elasticsearch version requirements + +> Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0. + +Advanced Search works with the following versions of Elasticsearch. + +| GitLab version | Elasticsearch version | +|-----------------------|--------------------------| +| GitLab 15.0 or later | Elasticsearch 7.x - 8.x | +| GitLab 13.9 - 14.10 | Elasticsearch 6.8 - 7.x | +| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x | +| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x | + +Advanced Search follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol). +When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts +before we remove them. + +### OpenSearch version requirements + +| GitLab version | Elasticsearch version | +|-----------------------|--------------------------| +| GitLab 15.0 or later | OpenSearch 1.x or later | + +If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing). + +## System requirements + +Elasticsearch requires additional resources to those documented in the +[GitLab system requirements](../../install/requirements.md). + +Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to +[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), +each node should have: + +- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). +- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs. +- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements. + +## Install Elasticsearch + +Elasticsearch is *not* included in the Omnibus packages or when you install from +source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. + +You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html) +service. + +You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. + +For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards. + +The search index updates after you: + +- Add data to the database or repository. +- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area. + +## Upgrade to a new Elasticsearch major version + +> - Elasticsearch 6.8 support is removed with GitLab 15.0. +> - Upgrading from GitLab 14.10 to 15.0 requires that you are using any version of Elasticsearch 7.x. + +You are not required to change the GitLab configuration when you upgrade Elasticsearch. + +## Elasticsearch repository indexer + +To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). + +Depending on your GitLab version, there are different installation procedures for the Go indexer: + +- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab). +- For installations from source or older versions of Omnibus GitLab, + [install the indexer from source](#from-source). +- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md). + +### Omnibus GitLab + +Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab. +The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481). + +### From source + +First, we need to install some dependencies, then we build and install +the indexer itself. + +This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding, +therefore we must ensure the development packages for your platform are +installed before running `make`. + +#### Debian / Ubuntu + +To install on Debian or Ubuntu, run: + +```shell +sudo apt install libicu-dev +``` + +#### CentOS / RHEL + +To install on CentOS or RHEL, run: + +```shell +sudo yum install libicu-devel +``` + +#### macOS + +NOTE: +You must first [install Homebrew](https://brew.sh/). + +To install on macOS, run: + +```shell +brew install icu4c +export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" +``` + +### Build and install + +To build and install the indexer, run: + +```shell +indexer_path=/home/git/gitlab-elasticsearch-indexer + +# Run the installation task for gitlab-elasticsearch-indexer: +sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production +cd $indexer_path && sudo make install +``` + +The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`. + +You can change the installation path with the `PREFIX` environment variable. +Please remember to pass the `-E` flag to `sudo` if you do so. + +Example: + +```shell +PREFIX=/usr sudo -E make install +``` + +After installation, be sure to [enable Elasticsearch](#enable-advanced-search). + +NOTE: +If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you +may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to +`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. + +## Enable Advanced Search + +For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. + +To enable Advanced Search, you must have administrator access to GitLab: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. + + NOTE: + To see the Advanced Search section, you need an active GitLab Premium + [license](../../user/admin_area/license.md). + +1. Configure the [Advanced Search settings](#advanced-search-configuration) for + your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** + yet. +1. Enable **Elasticsearch indexing** and select **Save changes**. This creates + an empty index if one does not already exist. +1. Select **Index all projects**. +1. Select **Check progress** in the confirmation message to see the status of + the background jobs. +1. Personal snippets must be indexed using another Rake task: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_snippets + + # Installations from source + bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production + ``` + +1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**. + +NOTE: +When your Elasticsearch cluster is down while Elasticsearch is enabled, +you might have problems updating documents such as issues because your +instance queues a job to index the change, but cannot find a valid +Elasticsearch cluster. + +### Advanced Search configuration + +The following Elasticsearch settings are available: + +| Parameter | Description | +|-------------------------------------------------------|-------------| +| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | +| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. | +| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | +| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. | +| `Username` | The `username` of your Elasticsearch instance. | +| `Password` | The password of your Elasticsearch instance. | +| `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | +| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | +| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). +| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | +| `AWS Region` | The AWS region in which your OpenSearch Service is located. | +| `AWS Access Key` | The AWS access key. | +| `AWS Secret Access Key` | The AWS secret access key. | +| `Maximum file size indexed` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-file-size-indexed). | +| `Maximum field length` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-field-length). | +| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. | + +WARNING: +Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact +Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations +in your Sidekiq logs. For more information, see +[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). + +### Limit the number of namespaces and projects that can be indexed + +If you check checkbox `Limit the number of namespaces and projects that can be indexed` +under **Elasticsearch indexing restrictions** more options become available. + + + +You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes +any subgroups and projects belonging to those subgroups to be indexed as well. + +Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. + +You can filter the selection dropdown by writing part of the namespace or project name you're interested in. + + + +NOTE: +If no namespaces or projects are selected, no Advanced Search indexing takes place. + +WARNING: +If you have already indexed your instance, you must regenerate the index to delete all existing data +for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and +`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data +from the Elasticsearch index as expected. + +## Enable custom language analyzers + +You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic. + +To enable languages support: + +1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Locate **Custom analyzers: language support**. +1. Enable plugins support for **Indexing**. +1. Select **Save changes** for the changes to take effect. +1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings. +1. Enable plugins support for **Searching** after the previous step is completed. + +For guidance on what to install, see the following Elasticsearch language plugin options: + +| Parameter | Description | +|-------------------------------------------------------|-------------| +| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.| +| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| +| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.| +| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| + +## Disable Advanced Search + +To disable the Elasticsearch integration: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. +1. Select **Save changes**. +1. Optional. Delete the existing indices: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:delete_index + + # Installations from source + bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production + ``` + +## Unpause Indexing + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Advanced Search**. +1. Clear the **Pause Elasticsearch indexing** checkbox. + +## Zero downtime reindexing + +The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) +and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a +`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause +the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the +index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the +index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes. + +### Trigger the reindex via the Advanced Search administration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. +> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. +> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. + +To trigger the reindexing process: + +1. Sign in to your GitLab instance as an administrator. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Elasticsearch zero-downtime reindexing**. +1. Select **Trigger cluster reindexing**. + +Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. + +After this process is completed, the original index is scheduled to be deleted after +14 days. You can cancel this action by pressing the **Cancel** button on the same +page you triggered the reindexing process. + +While the reindexing is running, you can follow its progress under that same section. + +#### Elasticsearch zero-downtime reindexing + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Elasticsearch zero-downtime reindexing**, and you'll + find the following options: + +- [Slice multiplier](#slice-multiplier) +- [Maximum running slices](#maximum-running-slices) + +##### Slice multiplier + +The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice). + +GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice) +to control the reindex efficiently and safely, which enables users to retry only +failed slices. + +The multiplier defaults to `2` and applies to the number of shards per index. +For example, if this value is `2` and your index has 20 shards, then the +reindex task is split into 40 slices. + +##### Maximum running slices + +The maximum running slices parameter defaults to `60` and corresponds to the +maximum number of slices allowed to run concurrently during Elasticsearch +reindexing. + +Setting this value too high can have adverse performance impacts as your cluster +may become heavily saturated with searches and writes. Setting this value too +low may lead the reindexing process to take a very long time to complete. + +The best value for this depends on your cluster size, whether you're willing +to accept some degraded search performance during reindexing, and how important +it is for the reindex to finish quickly and resume indexing. + +### Mark the most recent reindex job as failed and resume the indexing + +Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps: + +1. Mark the most recent reindex job as failed: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:mark_reindex_failed + + # Installations from source + bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production + ``` + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > Advanced Search**. +1. Expand **Advanced Search**. +1. Clear the **Pause Elasticsearch indexing** checkbox. + +## Advanced Search migrations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6. + +With reindex migrations running in the background, there's no need for a manual +intervention. This usually happens in situations where new features are added to +Advanced Search, which means adding or changing the way content is indexed. + +To confirm that the Advanced Search migrations ran, you can check with: + +```shell +curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq . +``` + +This should return something similar to: + +```json +{ + "took": 14, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1, + "hits": [ + { + "_index": "gitlab-production-migrations", + "_type": "_doc", + "_id": "20201105181100", + "_score": 1, + "_source": { + "completed": true + } + } + ] + } +} +``` + +In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog). + +### Retry a halted migration + +Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, +it is halted and a notification is displayed in the Advanced Search integration settings. +It is recommended to check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog) to +debug why the migration was halted and make any changes before retrying the migration. Once you believe you've +fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried +in the background. + +If you cannot get the migration to succeed, you may +consider the +[last resort to recreate the index from scratch](elasticsearch_troubleshooting.md#last-resort-to-recreate-an-index). +This may allow you to skip over +the problem because a newly created index skips all migrations as the index +is recreated with the correct up-to-date schema. + +### All migrations must be finished before doing a major upgrade + +Before doing a major version GitLab upgrade, you should have completed all +migrations that exist up until the latest minor version before that major +version. If you have halted migrations, these need to be resolved and +[retried](#retry-a-halted-migration) before proceeding with a major version +upgrade. Read more about [upgrading to a new major +version](../../update/index.md#upgrading-to-a-new-major-version). + +## GitLab Advanced Search Rake tasks + +Rake tasks are available to: + +- [Build and install](#build-and-install) the indexer. +- Delete indices when [disabling Elasticsearch](#disable-advanced-search). +- Add GitLab data to an index. + +The following are some available Rake tasks: + +| Task | Description | +|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | +| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | +| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | +| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. | +| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | +| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. | +| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indices (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. | +| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indices and aliases (if they exist) on the Elasticsearch instance. | +| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. | +| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | +| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | +| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. | +| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. | +| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. | +| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. | +| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. | +| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. | + +### Environment variables + +In addition to the Rake tasks, there are some environment variables that can be used to modify the process: + +| Environment Variable | Data Type | What it does | +| -------------------- |:---------:| ---------------------------------------------------------------------------- | +| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). | +| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | +| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | + +### Indexing a range of projects or a specific project + +Using the `ID_FROM` and `ID_TO` environment variables, you can index a limited number of projects. This can be useful for staging indexing. + +```shell +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1 ID_TO=100 +``` + +Because `ID_FROM` and `ID_TO` use the `or equal to` comparison, you can use them to index only one project +by setting both to the same project ID: + +```shell +root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=5 ID_TO=5 +Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... +I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! +``` + +## Advanced Search index scopes + +When performing a search, the GitLab index uses the following scopes: + +| Scope Name | What it searches | +| ---------------- | ---------------------- | +| `commits` | Commit data | +| `projects` | Project data (default) | +| `blobs` | Code | +| `issues` | Issue data | +| `merge_requests` | Merge request data | +| `milestones` | Milestone data | +| `notes` | Note data | +| `snippets` | Snippet data | +| `wiki_blobs` | Wiki contents | + +## Tuning + +### Guidance on choosing optimal cluster configuration + +For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below. + +- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. +- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). +- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. +- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details. +- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. +- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health. +- Number of Elasticsearch shards: + - Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB. + - Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example: + - If you have fewer than about 2,000,000 documents, use the default of 5 shards + - 10,000,000 documents: `10000000/5000000 + 5` = 7 shards + - 100,000,000 documents: `100000000/5000000 + 5` = 25 shards +- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value. +- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. + +### Advanced Search integration settings guidance + +- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. +- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index. + +### How to index large instances efficiently + +This section may be helpful in the event that the other +[basic instructions](#enable-advanced-search) cause problems +due to large volumes of data being indexed. + +WARNING: +Indexing a large instance generates a lot of Sidekiq jobs. +Make sure to prepare for this task by having a [Scalable and Highly Available +Setup](../../administration/reference_architectures/index.md) or creating [extra +Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). + +1. [Configure your Elasticsearch host and port](#enable-advanced-search). +1. Create empty indices: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:create_empty_index + + # Installations from source + bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production + ``` + +1. If this is a re-index of your GitLab instance, clear the index status: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:clear_index_status + + # Installations from source + bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production + ``` + +1. [Enable **Elasticsearch indexing**](#enable-advanced-search). +1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): + + - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. + + - You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete. + + In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings. + + NOTE: + This step is optional but may help significantly speed up large indexing operations. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "refresh_interval" : "-1", + "number_of_replicas" : 0 + } }' + ``` + +1. Index projects and their associated data: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects + + # Installations from source + bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production + ``` + + This enqueues a Sidekiq job for each project that needs to be indexed. + You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab** + and select `elastic_commit_indexer`, or you can query indexing status using a Rake task: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects_status + + # Installations from source + bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production + + Indexing is 65.55% complete (6555/10000 projects) + ``` + + If you want to limit the index to a range of projects you can provide the + `ID_FROM` and `ID_TO` parameters: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 + + # Installations from source + bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production + ``` + + Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional. + The above example will index all projects from ID `1001` up to (and including) ID `2000`. + + NOTE: + Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects` + can get interrupted. This may happen for many reasons, but it's always safe + to run the indexing task again. It will skip repositories that have + already been indexed. + + As the indexer stores the last commit SHA of every indexed repository in the + database, you can run the indexer with the special parameter `UPDATE_INDEX` and + it will check every project repository again to make sure that every commit in + a repository is indexed, which can be useful in case if your index is outdated: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 + + # Installations from source + bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production + ``` + + You can also use the `gitlab:elastic:clear_index_status` Rake task to force the + indexer to "forget" all progress, so it retries the indexing process from the + start. + +1. Personal snippets are not associated with a project and need to be indexed separately: + + ```shell + # Omnibus installations + sudo gitlab-rake gitlab:elastic:index_snippets + + # Installations from source + bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production + ``` + +1. Enable replication and refreshing again after indexing (only if you previously disabled it): + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "number_of_replicas" : 1, + "refresh_interval" : "1s" + } }' + ``` + + A force merge should be called after enabling the refreshing above. + + For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge: + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "settings": { + "index.blocks.write": true + } }' + ``` + + Then, initiate the force merge: + + ```shell + curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5' + ``` + + After this, if your index is in read-only mode, switch back to read-write: + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "settings": { + "index.blocks.write": false + } }' + ``` + +1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search). + +### Deleted documents + +Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index. + +In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_ + +However, some larger installations may wish to tune the merge policy settings: + +- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ + --data '{ + "index" : { + "merge.policy.max_merged_segment": "2gb" + } + }' + ``` + +- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. + + ```shell + curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ + --data '{ + "index" : { + "merge.policy.reclaim_deletes_weight": "3.0" + } + }' + ``` + +- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues. + +## Index large instances with dedicated Sidekiq nodes or processes + +Indexing a large instance can be a lengthy and resource-intensive process that has the potential +of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and +availability. + +As GitLab allows you to start multiple Sidekiq processes, you can create an +additional process dedicated to indexing a set of queues (or queue group). This way, you can +ensure that indexing queues always have a dedicated worker, while the rest of the queues have +another dedicated worker to avoid contention. + +For this purpose, use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) +option that allows a more general selection of queue groups using a [worker matching query](../../administration/operations/extra_sidekiq_routing.md#worker-matching-query). + +To handle these two queue groups, we generally recommend one of the following two options. You can either: + +- [Use two queue groups on one single node](#single-node-two-processes). +- [Use two queue groups, one on each node](#two-nodes-one-process-for-each). + +For the steps below, consider: + +- `feature_category=global_search` as an indexing queue group with its own Sidekiq process. +- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process. + +### Single node, two processes + +To create both an indexing and a non-indexing Sidekiq process in one node: + +1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=global_search", + "feature_category!=global_search" + ] + ``` + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +WARNING: +When starting multiple processes, the number of processes cannot exceed the number of CPU +cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject +to the available workload and concurrency settings. For more details, see how to +[run multiple Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). + +### Two nodes, one process for each + +To handle these queue groups on two nodes: + +1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=global_search" + ] + ``` + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category!=global_search" + ] + ``` + + to set up a non-indexing Sidekiq process. + +1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) +for the changes to take effect. + +## Reverting to Basic Search + +Sometimes there may be issues with your Elasticsearch index data and as such +GitLab allows you to revert to "basic search" when there are no search +results and assuming that basic search is supported in that scope. This "basic +search" behaves as though you don't have Advanced Search enabled at all for +your instance and search using other data sources (such as PostgreSQL data and Git +data). + +## Data recovery: Elasticsearch is a secondary data store only + +The use of Elasticsearch in GitLab is only ever as a secondary data store. +This means that all of the data stored in Elasticsearch can always be derived +again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if +the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch. diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md new file mode 100644 index 00000000000..97abf456baa --- /dev/null +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -0,0 +1,274 @@ +--- +type: reference +stage: Data Stores +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Elasticsearch troubleshooting **(PREMIUM SELF)** + +Use the following information to troubleshoot Elasticsearch issues. + +## View logs + +One of the most valuable tools for identifying issues with the Elasticsearch +integration are logs. The most relevant logs for this integration are: + +1. [`sidekiq.log`](../../administration/logs.md#sidekiqlog) - All of the + indexing happens in Sidekiq, so much of the relevant logs for the + Elasticsearch integration can be found in this file. +1. [`elasticsearch.log`](../../administration/logs.md#elasticsearchlog) - There + are additional logs specific to Elasticsearch that are sent to this file + that may contain useful diagnostic information about searching, + indexing or migrations. + +Here are some common pitfalls and how to overcome them. + +## How can I verify that my GitLab instance is using Elasticsearch? + +There are a couple of ways to achieve that: + +- Whenever you perform a search there is a link on the search results page + in the top right hand corner saying "Advanced search functionality is enabled". + This is always correctly identifying whether the current project/namespace + being searched is using Elasticsearch. + +- From the Admin Area under **Settings > Advanced Search** check that the + Advanced Search settings are checked. + + Those same settings there can be obtained from the Rails console if necessary: + + ```ruby + ::Gitlab::CurrentSettings.elasticsearch_search? # Whether or not searches will use Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_indexing? # Whether or not content will be indexed in Elasticsearch + ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces + ``` + +- If Elasticsearch is limited to specific namespaces and you need to know if + Elasticsearch is being used for a specific project or namespace, you can use + the Rails console: + + ```ruby + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Namespace.find_by_full_path("/my-namespace")) + ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) + ``` + +## I updated GitLab and now I can't find anything + +We continuously make updates to our indexing strategies and aim to support +newer versions of Elasticsearch. When indexing changes are made, it may +be necessary for you to [reindex](elasticsearch.md#zero-downtime-reindexing) after updating GitLab. + +## I indexed all the repositories but I can't get any hits for my search term in the UI + +Make sure you [indexed all the database data](elasticsearch.md#enable-advanced-search). + +If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): + +```ruby +u = User.find_by_username('your-username') +s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) +pp s.search_objects.to_a +``` + +Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side: + +```shell +curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<search_term> +``` + +More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. + +It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../../administration/troubleshooting/elasticsearch.md#search-results-workflow). + +NOTE: +The above instructions are not to be used for scenarios that only index a [subset of namespaces](elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). + +See [Elasticsearch Index Scopes](elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. + +## I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything + +You must re-run all the Rake tasks to reindex the database, repositories, and wikis. + +## The indexing process is taking a very long time + +The more data present in your GitLab instance, the longer the indexing process takes. + +## There are some projects that weren't indexed, but I don't know which ones + +You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. + +## No new data is added to the Elasticsearch index when I push code + +NOTE: +This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. + +When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. To unlock them, run: + +```shell +sudo gitlab-rake gitlab:elastic:clear_locked_projects +``` + +## `Can't specify parent if no parent field has been configured` error + +If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indices, you get +exceptions in lots of different cases: + +```plaintext +Elasticsearch::Transport::Transport::Errors::BadRequest([400] { + "error": { + "root_cause": [{ + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }], + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }, + "status": 400 +}): +``` + +This is because we changed the index mapping in GitLab 8.12 and the old indices should be removed and built from scratch again, +see details in the [update guide](../../update/upgrading_from_source.md). + +## `Elasticsearch::Transport::Transport::Errors::BadRequest` + +If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](elasticsearch.md#system-requirements). +There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. + +## `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` + +```plaintext +[413] {"Message":"Request size exceeded 10485760 bytes"} +``` + +This exception is seen when your Elasticsearch cluster is configured to reject requests above a certain size (10MiB in this case). This corresponds to the `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a larger size and restart your Elasticsearch cluster. + +AWS has [fixed limits](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#network-limits) for this setting ("Maximum size of HTTP request payloads"), based on the size of the underlying instance. + +## My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly + +**For a single node Elasticsearch cluster the functional cluster health status is yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. + +WARNING: +Setting the number of replicas to `0` is discouraged (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas needs to be set to an integer value larger than `0`. Failure to do so results in lack of redundancy (losing one node corrupts the index). + +If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then run the following query to set the number of replicas to `0`(the cluster no longer tries to create any shard replicas): + +```shell +curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ + --data '{ + "index" : { + "number_of_replicas" : 0 + } + }' +``` + +## `health check timeout: no Elasticsearch node available` error in Sidekiq + +If you're getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process: + +```plaintext +Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" +``` + +You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). +After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](elasticsearch.md#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](elasticsearch.md#enable-advanced-search). + +## My Elasticsearch cluster has a plugin and the integration is not working + +Certain 3rd party plugins may introduce bugs in your cluster or for whatever +reason may be incompatible with our integration. You should try disabling +plugins so you can rule out the possibility that the plugin is causing the +problem. + +## Low-level troubleshooting + +There is a [more structured, lower-level troubleshooting document](../../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. + +## Elasticsearch `code_analyzer` doesn't account for all code cases + +The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. + +Improvements to the `code_analyzer` pattern and filters are being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). + +## Some binary files may not be searchable by name + +In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. + +## Last resort to recreate an index + +There may be cases where somehow data never got indexed and it's not in the +queue, or the index is somehow in a state where migrations just cannot +proceed. It is always best to try to troubleshoot the root cause of the problem +by [viewing the logs](#view-logs). + +If there are no other options, then you always have the option of recreating the +entire index from scratch. If you have a small GitLab installation, this can +sometimes be a quick way to resolve a problem, but if you have a large GitLab +installation, then this might take a very long time to complete. Until the +index is fully recreated, your index does not serve correct search results, +so you may want to disable **Search with Elasticsearch** while it is running. + +If you are sure you've read the above caveats and want to proceed, then you +should run the following Rake task to recreate the entire index from scratch: + +**For Omnibus installations** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +sudo gitlab-rake gitlab:elastic:index +``` + +**For installations from source** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:elastic:index +``` + +## How does Advanced Search handle private projects? + +Advanced Search stores all the projects in the same Elasticsearch indices, +however, searches only surface results that can be viewed by the user. +Advanced Search honors all permission checks in the application by +filtering out projects that a user does not have access to at search time. + +## Indexing fails with `error: elastic: Error 429 (Too Many Requests)` + +If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: + +- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. +- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. + +## Indexing is very slow or fails with `rejected execution of coordinating operation` messages + +Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. +Ensure that your Elasticsearch cluster meets the [system requirements](elasticsearch.md#system-requirements) and has enough resources +to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). + +## Access requirements for the self-managed AWS OpenSearch Service + +To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies +to contain the actions below. +See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. + +```plaintext +es:ESHttpDelete +es:ESHttpGet +es:ESHttpHead +es:ESHttpPost +es:ESHttpPut +es:ESHttpPatch +``` + +## Role-mapping when using AWS Elasticsearch or AWS OpenSearch fine-grained access control + +When using fine-grained access control with an IAM role, you might encounter the following error: + +```plaintext +{"error":{"root_cause":[{"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"}],"type":"security_exception","reason":"no permissions for [indices:data/write/bulk] and User [name=arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE, backend_roles=[arn:aws:iam::xxx:role/INSERT_ROLE_NAME_HERE], requestedTenant=null]"},"status":403} +``` + +To fix this, you need to [map the roles to users](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-mapping) in Kibana. diff --git a/doc/integration/img/limit_namespace_filter.png b/doc/integration/advanced_search/img/limit_namespace_filter.png Binary files differindex 437aecad467..437aecad467 100644 --- a/doc/integration/img/limit_namespace_filter.png +++ b/doc/integration/advanced_search/img/limit_namespace_filter.png diff --git a/doc/integration/img/limit_namespaces_projects_options.png b/doc/integration/advanced_search/img/limit_namespaces_projects_options.png Binary files differindex a3cf9933b7e..a3cf9933b7e 100644 --- a/doc/integration/img/limit_namespaces_projects_options.png +++ b/doc/integration/advanced_search/img/limit_namespaces_projects_options.png diff --git a/doc/integration/arkose.md b/doc/integration/arkose.md new file mode 100644 index 00000000000..0135785dc11 --- /dev/null +++ b/doc/integration/arkose.md @@ -0,0 +1,77 @@ +--- +stage: Anti-Abuse +group: Anti-Abuse +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Arkose Protect + +DISCLAIMER: +Arkose Protect is used on GitLab.com and is not supported for self-managed GitLab +instances. The following documents the internal requirements for maintaining +Arkose Protect on GitLab.com. While this feature is theoretically usable in self-managed instances, it +is not recommended at the moment. + +GitLab integrates [Arkose Protect](https://www.arkoselabs.com/arkose-protect/) to guard against +credential stuffing and bots in the sign-in form. GitLab will trigger Arkose Protect if the user: + +- Has never signed in before. +- Has failed to sign in twice in a row. +- Has not signed in during the past three months. + +## How does it work? + +If Arkose Protect determines that the user is suspicious, it presents an interactive challenge below +the `Sign in` button. The challenge needs to be completed to proceed with the sign-in +attempt. If Arkose Protect trusts the user, the challenge runs in transparent mode, meaning that the +user doesn't need to take any additional action and can sign in as usual. + +## How do we treat malicious sign-in attempts? + +Users are not denied access if Arkose Protect considers they are malicious. However, +their risk score is exposed in the admin console so that we can make more informed decisions when it +comes to manually blocking users. When we decide to block a user, feedback is sent to ArkoseLabs to +improve their risk prediction model. + +NOTE: +Enabling the `arkose_labs_prevent_login` feature flag results in sessions with a `High` risk +score being denied access. So far, we have kept this feature flag disabled to evaluate Arkose +Protect's predictions and to make sure we are not preventing legitimate users from signing in. + +That said, we have seen that interactive challenges are effective in preventing some malicious +sign-in attempts as not completing them prevents attackers from moving on to the next sign-in step. + +## Configuration + +To enable Arkose Protect: + +1. License ArkoseLabs. +1. Get the public and private API keys from the [ArkoseLabs Portal](https://portal.arkoselabs.com/). +1. Enable the ArkoseLabs login challenge. Run the following commands in the Rails console, replacing `<your_public_api_key>` and `<your_private_api_key>` with your own API keys. + + ```ruby + Feature.enable(:arkose_labs_login_challenge) + ApplicationSetting.current.update(arkose_labs_public_api_key: '<your_public_api_key>') + ApplicationSetting.current.update(arkose_labs_private_api_key: '<your_private_api_key>') + ``` + +1. Optional. To prevent high risk sessions from signing, enable the `arkose_labs_prevent_login` feature flag. Run the following command in the Rails console: + + ```ruby + Feature.enable(:arkose_labs_prevent_login) + ``` + +## QA tests caveat + +Several GitLab QA test suites need to sign in to the app to test its features. This can conflict +with Arkose Protect as it would identify QA users as being malicious because they are being run with +a headless browser. To work around this, ArkoseLabs has allowlisted the unique token +that serves as QA session's User Agent. While this doesn't guarantee that the session won't be +flagged as malicious, Arkose's API returns a specific telltale when we verify the sign in +attempt's token. We are leveraging this telltale to bypass the verification step entirely so that the +test suite doesn't fail. This bypass is done in the `UserVerificationService` class. + +## Feedback Job + +To help Arkose improve their protection service, we created a daily background job to send them the list of blocked users by us. +This job is performed by the `Arkose::BlockedUsersReportWorker` class. diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index 1e1ee9aebc5..71c71bd8b5c 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -97,6 +97,6 @@ application. [restart GitLab](../administration/restart_gitlab.md#installations-from-source). On the sign-in page there should now be an Auth0 icon below the regular sign-in -form. Click the icon to begin the authentication process. Auth0 asks the +form. Select the icon to begin the authentication process. Auth0 asks the user to sign in and authorize the GitLab application. If the user authenticates successfully, the user is returned to GitLab and signed in. diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 165d25c0d90..43032902a21 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -114,7 +114,7 @@ to the end of the Bitbucket authorization callback URL. if you installed from source. On the sign-in page there should now be a Bitbucket icon below the regular -sign-in form. Click the icon to begin the authentication process. Bitbucket asks +sign-in form. Select the icon to begin the authentication process. Bitbucket asks the user to sign in and authorize the GitLab application. If successful, the user is returned to GitLab and signed in. diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index ab9aaae63a2..0b34f7018da 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -1,1112 +1,11 @@ --- -type: reference -stage: Enablement -group: Global Search -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'advanced_search/elasticsearch.md' +remove_date: '2022-09-13' --- -# Elasticsearch integration **(PREMIUM SELF)** +This document was moved to [another location](advanced_search/elasticsearch.md). -> Moved to GitLab Premium in 13.9. - -This page describes how to enable Advanced Search. When enabled, -Advanced Search provides faster search response times and [improved search features](../user/search/advanced_search.md). - -## Version requirements - -> Support for Elasticsearch 6.8 was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 14.8 and is scheduled for removal in GitLab 15.0. - -| GitLab version | Elasticsearch version | -|----------------------|--------------------------| -| GitLab 14.8 or later | Elasticsearch 7.x - 7.17 | -| GitLab 13.9 - 14.7 | Elasticsearch 6.8 - 7.x | -| GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x | -| GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x | -| GitLab 11.5 - 12.6 | Elasticsearch 5.6 - 6.x | - -The Elasticsearch Integration works with supported versions of -Elasticsearch and follows Elasticsearch's [End of Life Policy](https://www.elastic.co/support/eol). -When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts -before we remove them. - -### Versions not supported - -GitLab does not support: - -- [Amazon's OpenSearch](https://aws.amazon.com/blogs/opensource/opensearch-1-0-launches/) -(a [fork of Elasticsearch](https://www.elastic.co/what-is/opensearch)). Use AWS Elasticsearch Service 7.10 instead. -For updates, see [issue #327560](https://gitlab.com/gitlab-org/gitlab/-/issues/327560). -- Elasticsearch 8.0. For updates, see [issue #350600](https://gitlab.com/gitlab-org/gitlab/-/issues/350600). Use Elasticsearch 7.17 instead. - -If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing). - -## System requirements - -Elasticsearch requires additional resources to those documented in the -[GitLab system requirements](../install/requirements.md). - -Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. According to -[Elasticsearch official guidelines](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory), -each node should have: - -- [Memory](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_memory): 8 GiB (minimum). -- [CPU](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_cpus): Modern processor with multiple cores. GitLab.com has minimal CPU requirements for Elasticsearch. Multiple cores provide extra concurrency, which is more beneficial than faster CPUs. -- [Storage](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html#_disks): Use SSD storage. The total storage size of all Elasticsearch nodes is about 50% of the total size of your Git repositories. It includes one primary and one replica. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses total repository size to estimate the Advanced Search storage requirements. - -## Install Elasticsearch - -Elasticsearch is *not* included in the Omnibus packages or when you install from -source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. - -You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html) -service. - -If using the Amazon OpenSearch service, ensure that you select `Elasticsearch 7.10` when configuring Deployment type. As noted in [Versions not supported](#versions-not-supported), Amazon's non-Elasticsearch versions are not yet supported. - -You should install Elasticsearch on a separate server. Running Elasticsearch on the same server as GitLab is not recommended and can cause a degradation in GitLab instance performance. - -For a single node Elasticsearch cluster, the functional cluster health status is always yellow due to the allocation of the primary shard. Elasticsearch cannot assign replica shards to the same node as primary shards. - -The search index updates after you: - -- Add data to the database or repository. -- [Enable Elasticsearch](#enable-advanced-search) in the Admin Area. - -## Upgrade to a new Elasticsearch major version - -Elasticsearch reads and uses indices created in the previous major version. You are not required to change the GitLab configuration when you upgrade Elasticsearch. - -If your current index was created before GitLab 13.0, you must reindex from scratch to create an alias to use features such as [zero downtime reindexing](#zero-downtime-reindexing). After you reindex, you can perform zero downtime reindexing and also benefit from future features that use the alias. - -To check if your current index was created before GitLab 13.0, use the [Elasticsearch cat aliases API](https://www.elastic.co/guide/en/elasticsearch/reference/7.11/cat-alias.html). -If the returned list of aliases does not contain a `gitlab-production` alias, you must reindex to use features such as zero downtime reindexing. -If the returned list of aliases contains an entry for `gitlab-production` that points to an index -named `gitlab-production-<numerical timestamp>`, your index was created after GitLab 13.0. - -## Elasticsearch repository indexer - -To index Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). - -Depending on your GitLab version, there are different installation procedures for the Go indexer: - -- For Omnibus GitLab 11.8 or greater, see [Omnibus GitLab](#omnibus-gitlab). -- For installations from source or older versions of Omnibus GitLab, - [install the indexer from source](#from-source). -- If you are using GitLab Development Kit, see [GDK Elasticsearch how-to](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/elasticsearch.md). - -### Omnibus GitLab - -Starting with GitLab 11.8, the Go indexer is included in Omnibus GitLab. -The former Ruby-based indexer was removed in [GitLab 12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/6481). - -### From source - -First, we need to install some dependencies, then we build and install -the indexer itself. - -This project relies on [International Components for Unicode](https://icu.unicode.org/) (ICU) for text encoding, -therefore we must ensure the development packages for your platform are -installed before running `make`. - -#### Debian / Ubuntu - -To install on Debian or Ubuntu, run: - -```shell -sudo apt install libicu-dev -``` - -#### CentOS / RHEL - -To install on CentOS or RHEL, run: - -```shell -sudo yum install libicu-devel -``` - -#### macOS - -NOTE: -You must first [install Homebrew](https://brew.sh/). - -To install on macOS, run: - -```shell -brew install icu4c -export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" -``` - -### Build and install - -To build and install the indexer, run: - -```shell -indexer_path=/home/git/gitlab-elasticsearch-indexer - -# Run the installation task for gitlab-elasticsearch-indexer: -sudo -u git -H bundle exec rake gitlab:indexer:install[$indexer_path] RAILS_ENV=production -cd $indexer_path && sudo make install -``` - -The `gitlab-elasticsearch-indexer` is installed to `/usr/local/bin`. - -You can change the installation path with the `PREFIX` environment variable. -Please remember to pass the `-E` flag to `sudo` if you do so. - -Example: - -```shell -PREFIX=/usr sudo -E make install -``` - -After installation, be sure to [enable Elasticsearch](#enable-advanced-search). - -NOTE: -If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch-indexer/` while indexing, you -may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to -`/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. - -## Enable Advanced Search - -For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. - -To enable Advanced Search, you must have administrator access to GitLab: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. - - NOTE: - To see the Advanced Search section, you need an active GitLab Premium - [license](../user/admin_area/license.md). - -1. Configure the [Advanced Search settings](#advanced-search-configuration) for - your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** - yet. -1. Enable **Elasticsearch indexing** and select **Save changes**. This creates - an empty index if one does not already exist. -1. Select **Index all projects**. -1. Select **Check progress** in the confirmation message to see the status of - the background jobs. -1. Personal snippets must be indexed using another Rake task: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_snippets - - # Installations from source - bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production - ``` - -1. After indexing completes, enable **Search with Elasticsearch enabled** and select **Save changes**. - -NOTE: -When your Elasticsearch cluster is down while Elasticsearch is enabled, -you might have problems updating documents such as issues because your -instance queues a job to index the change, but cannot find a valid -Elasticsearch cluster. - -### Advanced Search configuration - -The following Elasticsearch settings are available: - -| Parameter | Description | -|-------------------------------------------------------|-------------| -| `Elasticsearch indexing` | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | -| `Pause Elasticsearch indexing` | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. | -| `Search with Elasticsearch enabled` | Enables or disables using Elasticsearch in search. | -| `URL` | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. | -| `Username` | The `username` of your Elasticsearch instance. | -| `Password` | The password of your Elasticsearch instance. | -| `Number of Elasticsearch shards` | Elasticsearch indexes are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indexes with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | -| `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | -| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). -| `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | -| `AWS Region` | The AWS region in which your OpenSearch Service is located. | -| `AWS Access Key` | The AWS access key. | -| `AWS Secret Access Key` | The AWS secret access key. | -| `Maximum file size indexed` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-file-size-indexed). | -| `Maximum field length` | See [the explanation in instance limits.](../administration/instance_limits.md#maximum-field-length). | -| `Maximum bulk request size (MiB)` | The Maximum Bulk Request size is used by the GitLab Golang-based indexer processes and indicates how much data it ought to collect (and store in memory) in a given indexing process before submitting the payload to Elasticsearch's Bulk API. This setting should be used with the Bulk request concurrency setting (see below) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | -| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to Elasticsearch's Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | -| `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. | - -WARNING: -Increasing the values of `Maximum bulk request size (MiB)` and `Bulk request concurrency` can negatively impact -Sidekiq performance. Return them to their default values if you see increased `scheduling_latency_s` durations -in your Sidekiq logs. For more information, see -[issue 322147](https://gitlab.com/gitlab-org/gitlab/-/issues/322147). - -### Limit the number of namespaces and projects that can be indexed - -If you check checkbox `Limit the number of namespaces and projects that can be indexed` -under **Elasticsearch indexing restrictions** more options become available. - - - -You can select namespaces and projects to index exclusively. Note that if the namespace is a group, it includes -any subgroups and projects belonging to those subgroups to be indexed as well. - -Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. - -You can filter the selection dropdown by writing part of the namespace or project name you're interested in. - - - -NOTE: -If no namespaces or projects are selected, no Advanced Search indexing takes place. - -WARNING: -If you have already indexed your instance, you must regenerate the index to delete all existing data -for filtering to work correctly. To do this, run the Rake tasks `gitlab:elastic:recreate_index` and -`gitlab:elastic:clear_index_status`. Afterwards, removing a namespace or a project from the list deletes the data -from the Elasticsearch index as expected. - -## Enable custom language analyzers - -You can improve the language support for Chinese and Japanese languages by utilizing [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) and/or [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) analysis plugins from Elastic. - -To enable languages support: - -1. Install the desired plugins, please refer to [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/plugins/7.9/installation.html) for plugins installation instructions. The plugins must be installed on every node in the cluster, and each node must be restarted after installation. For a list of plugins, see the table later in this section. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Locate **Custom analyzers: language support**. -1. Enable plugins support for **Indexing**. -1. Click **Save changes** for the changes to take effect. -1. Trigger [Zero downtime reindexing](#zero-downtime-reindexing) or reindex everything from scratch to create a new index with updated mappings. -1. Enable plugins support for **Searching** after the previous step is completed. - -For guidance on what to install, see the following Elasticsearch language plugin options: - -| Parameter | Description | -|-------------------------------------------------------|-------------| -| `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.| -| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| -| `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.| -| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Please only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| - -## Disable Advanced Search - -To disable the Elasticsearch integration: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. -1. Select **Save changes**. -1. Optional. Delete the existing indexes: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:delete_index - - # Installations from source - bundle exec rake gitlab:elastic:delete_index RAILS_ENV=production - ``` - -## Unpause Indexing - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Advanced Search**. -1. Clear the **Pause Elasticsearch indexing** checkbox. - -## Zero downtime reindexing - -The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) -and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a -`primary` index which is used by GitLab for reads/writes. When reindexing process starts, we temporarily pause -the writes to the `primary` index. Then, we create another index and invoke the Reindex API which migrates the -index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the -index alias to it which becomes the new `primary` index. At the end, we resume the writes and normal operation resumes. - -### Trigger the reindex via the Advanced Search administration - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. -> - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. -> - Support for retries during reindexing was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. - -To trigger the reindexing process: - -1. Sign in to your GitLab instance as an administrator. -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Elasticsearch zero-downtime reindexing**. -1. Select **Trigger cluster reindexing**. - -Reindexing can be a lengthy process depending on the size of your Elasticsearch cluster. - -After this process is completed, the original index is scheduled to be deleted after -14 days. You can cancel this action by pressing the **Cancel** button on the same -page you triggered the reindexing process. - -While the reindexing is running, you can follow its progress under that same section. - -#### Elasticsearch zero-downtime reindexing - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55681) in GitLab 13.12. - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Elasticsearch zero-downtime reindexing**, and you'll - find the following options: - -- [Slice multiplier](#slice-multiplier) -- [Maximum running slices](#maximum-running-slices) - -##### Slice multiplier - -The slice multiplier calculates the [number of slices during reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-slice). - -GitLab uses [manual slicing](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#docs-reindex-manual-slice) -to control the reindex efficiently and safely, which enables users to retry only -failed slices. - -The multiplier defaults to `2` and applies to the number of shards per index. -For example, if this value is `2` and your index has 20 shards, then the -reindex task is split into 40 slices. - -##### Maximum running slices - -The maximum running slices parameter defaults to `60` and corresponds to the -maximum number of slices allowed to run concurrently during Elasticsearch -reindexing. - -Setting this value too high can have adverse performance impacts as your cluster -may become heavily saturated with searches and writes. Setting this value too -low may lead the reindexing process to take a very long time to complete. - -The best value for this depends on your cluster size, whether you're willing -to accept some degraded search performance during reindexing, and how important -it is for the reindex to finish quickly and resume indexing. - -### Mark the most recent reindex job as failed and resume the indexing - -Sometimes, you might want to abandon the unfinished reindex job and resume the indexing. You can achieve this via the following steps: - -1. Mark the most recent reindex job as failed: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:mark_reindex_failed - - # Installations from source - bundle exec rake gitlab:elastic:mark_reindex_failed RAILS_ENV=production - ``` - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > Advanced Search**. -1. Expand **Advanced Search**. -1. Clear the **Pause Elasticsearch indexing** checkbox. - -## Advanced Search migrations - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6. - -With reindex migrations running in the background, there's no need for a manual -intervention. This usually happens in situations where new features are added to -Advanced Search, which means adding or changing the way content is indexed. - -To confirm that the Advanced Search migrations ran, you can check with: - -```shell -curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq . -``` - -This should return something similar to: - -```json -{ - "took": 14, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 1, - "hits": [ - { - "_index": "gitlab-production-migrations", - "_type": "_doc", - "_id": "20201105181100", - "_score": 1, - "_source": { - "completed": true - } - } - ] - } -} -``` - -In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog). - -### Retry a halted migration - -Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, -it is halted and a notification is displayed in the Advanced Search integration settings. -It is recommended to check the [`elasticsearch.log` file](../administration/logs.md#elasticsearchlog) to -debug why the migration was halted and make any changes before retrying the migration. Once you believe you've -fixed the cause of the failure, click "Retry migration", and the migration is scheduled to be retried -in the background. - -If you cannot get the migration to succeed, you may -consider the [last resort to recreate the index from -scratch](#last-resort-to-recreate-an-index). This may allow you to skip over -the problem because a newly created index skips all migrations as the index -is recreated with the correct up-to-date schema. - -### All migrations must be finished before doing a major upgrade - -Before doing a major version GitLab upgrade, you should have completed all -migrations that exist up until the latest minor version before that major -version. If you have halted migrations, these need to be resolved and -[retried](#retry-a-halted-migration) before proceeding with a major version -upgrade. Read more about [upgrading to a new major -version](../update/index.md#upgrading-to-a-new-major-version). - -## GitLab Advanced Search Rake tasks - -Rake tasks are available to: - -- [Build and install](#build-and-install) the indexer. -- Delete indexes when [disabling Elasticsearch](#disable-advanced-search). -- Add GitLab data to an index. - -The following are some available Rake tasks: - -| Task | Description | -|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, and `gitlab:elastic:index_snippets`. | -| [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | -| [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | -| [`sudo gitlab-rake gitlab:elastic:index_projects`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Iterates over all projects, and queues Sidekiq jobs to index them in the background. It can only be used after the index is created. | -| [`sudo gitlab-rake gitlab:elastic:index_projects_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Determines the overall status of the indexing. It is done by counting the total number of indexed projects, dividing by a count of the total number of projects, then multiplying by 100. | -| [`sudo gitlab-rake gitlab:elastic:clear_index_status`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Deletes all instances of IndexStatus for all projects. Note that this command results in a complete wipe of the index, and it should be used with caution. | -| [`sudo gitlab-rake gitlab:elastic:create_empty_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Generates empty indexes (the default index and a separate issues index) and assigns an alias for each on the Elasticsearch side only if it doesn't already exist. | -| [`sudo gitlab-rake gitlab:elastic:delete_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Removes the GitLab indexes and aliases (if they exist) on the Elasticsearch instance. | -| [`sudo gitlab-rake gitlab:elastic:recreate_index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Wrapper task for `gitlab:elastic:delete_index` and `gitlab:elastic:create_empty_index`. | -| [`sudo gitlab-rake gitlab:elastic:index_snippets`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Performs an Elasticsearch import that indexes the snippets data. | -| [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Displays which projects are not indexed. | -| [`sudo gitlab-rake gitlab:elastic:reindex_cluster`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Schedules a zero-downtime cluster reindexing task. This feature should be used with an index that was created after GitLab 13.0. | -| [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. | -| [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. | -| [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. | -| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. | -| [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. | - -### Environment variables - -In addition to the Rake tasks, there are some environment variables that can be used to modify the process: - -| Environment Variable | Data Type | What it does | -| -------------------- |:---------:| ---------------------------------------------------------------------------- | -| `UPDATE_INDEX` | Boolean | Tells the indexer to overwrite any existing index data (true/false). | -| `ID_TO` | Integer | Tells the indexer to only index projects less than or equal to the value. | -| `ID_FROM` | Integer | Tells the indexer to only index projects greater than or equal to the value. | - -### Indexing a specific project - -Because the `ID_TO` and `ID_FROM` environment variables use the `or equal to` comparison, you can index only one project by using both these variables with the same project ID number: - -```shell -root@git:~# sudo gitlab-rake gitlab:elastic:index_projects ID_TO=5 ID_FROM=5 -Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : Indexing GitLab User / test (ID=33)... -I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! -``` - -## Advanced Search index scopes - -When performing a search, the GitLab index uses the following scopes: - -| Scope Name | What it searches | -| ---------------- | ---------------------- | -| `commits` | Commit data | -| `projects` | Project data (default) | -| `blobs` | Code | -| `issues` | Issue data | -| `merge_requests` | Merge request data | -| `milestones` | Milestone data | -| `notes` | Note data | -| `snippets` | Snippet data | -| `wiki_blobs` | Wiki contents | - -## Tuning - -### Guidance on choosing optimal cluster configuration - -For basic guidance on choosing a cluster configuration you may refer to [Elastic Cloud Calculator](https://cloud.elastic.co/pricing). You can find more information below. - -- Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. -- It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). -- You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. -- `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details. -- Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. -- A good guideline is to ensure you keep the number of shards per node below 20 per GB heap it has configured. A node with a 30GB heap should therefore have a maximum of 600 shards, but the further below this limit you can keep it the better. This generally helps the cluster stay in good health. -- Number of Elasticsearch shards: - - Small shards result in small segments, which increases overhead. Aim to keep the average shard size between at least a few GB and a few tens of GB. - - Another consideration is the number of documents. To determine the number of shards to use, sum the numbers in the **Menu > Admin > Dashboard > Statistics** pane (the number of documents to be indexed), divide by 5 million, and add 5. For example: - - If you have fewer than about 2,000,000 documents, use the default of 5 shards - - 10,000,000 documents: `10000000/5000000 + 5` = 7 shards - - 100,000,000 documents: `100000000/5000000 + 5` = 25 shards -- `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value. -- You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. - -### Advanced Search integration settings guidance - -- The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. -- The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index. - -### How to index large instances efficiently - -This section may be helpful in the event that the other -[basic instructions](#enable-advanced-search) cause problems -due to large volumes of data being indexed. - -WARNING: -Indexing a large instance generates a lot of Sidekiq jobs. -Make sure to prepare for this task by having a [Scalable and Highly Available -Setup](../administration/reference_architectures/index.md) or creating [extra -Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). - -1. [Configure your Elasticsearch host and port](#enable-advanced-search). -1. Create empty indexes: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:create_empty_index - - # Installations from source - bundle exec rake gitlab:elastic:create_empty_index RAILS_ENV=production - ``` - -1. If this is a re-index of your GitLab instance, clear the index status: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:clear_index_status - - # Installations from source - bundle exec rake gitlab:elastic:clear_index_status RAILS_ENV=production - ``` - -1. [Enable **Elasticsearch indexing**](#enable-advanced-search). -1. Indexing large Git repositories can take a while. To speed up the process, you can [tune for indexing speed](https://www.elastic.co/guide/en/elasticsearch/reference/current/tune-for-indexing-speed.html#tune-for-indexing-speed): - - - You can temporarily disable [`refresh`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-refresh.html), the operation responsible for making changes to an index available to search. - - - You can set the number of replicas to 0. This setting controls the number of copies each primary shard of an index will have. Thus, having 0 replicas effectively disables the replication of shards across nodes, which should increase the indexing performance. This is an important trade-off in terms of reliability and query performance. It is important to remember to set the replicas to a considered value after the initial indexing is complete. - - In our experience, you can expect a 20% decrease in indexing time. After completing indexing in a later step, you can return `refresh` and `number_of_replicas` to their desired settings. - - NOTE: - This step is optional but may help significantly speed up large indexing operations. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "refresh_interval" : "-1", - "number_of_replicas" : 0 - } }' - ``` - -1. Index projects and their associated data: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects - - # Installations from source - bundle exec rake gitlab:elastic:index_projects RAILS_ENV=production - ``` - - This enqueues a Sidekiq job for each project that needs to be indexed. - You can view the jobs in **Menu > Admin > Monitoring > Background Jobs > Queues Tab** - and click `elastic_commit_indexer`, or you can query indexing status using a Rake task: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects_status - - # Installations from source - bundle exec rake gitlab:elastic:index_projects_status RAILS_ENV=production - - Indexing is 65.55% complete (6555/10000 projects) - ``` - - If you want to limit the index to a range of projects you can provide the - `ID_FROM` and `ID_TO` parameters: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 - - # Installations from source - bundle exec rake gitlab:elastic:index_projects ID_FROM=1001 ID_TO=2000 RAILS_ENV=production - ``` - - Where `ID_FROM` and `ID_TO` are project IDs. Both parameters are optional. - The above example will index all projects from ID `1001` up to (and including) ID `2000`. - - NOTE: - Sometimes the project indexing jobs queued by `gitlab:elastic:index_projects` - can get interrupted. This may happen for many reasons, but it's always safe - to run the indexing task again. It will skip repositories that have - already been indexed. - - As the indexer stores the last commit SHA of every indexed repository in the - database, you can run the indexer with the special parameter `UPDATE_INDEX` and - it will check every project repository again to make sure that every commit in - a repository is indexed, which can be useful in case if your index is outdated: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 - - # Installations from source - bundle exec rake gitlab:elastic:index_projects UPDATE_INDEX=true ID_TO=1000 RAILS_ENV=production - ``` - - You can also use the `gitlab:elastic:clear_index_status` Rake task to force the - indexer to "forget" all progress, so it retries the indexing process from the - start. - -1. Personal snippets are not associated with a project and need to be indexed separately: - - ```shell - # Omnibus installations - sudo gitlab-rake gitlab:elastic:index_snippets - - # Installations from source - bundle exec rake gitlab:elastic:index_snippets RAILS_ENV=production - ``` - -1. Enable replication and refreshing again after indexing (only if you previously disabled it): - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "number_of_replicas" : 1, - "refresh_interval" : "1s" - } }' - ``` - - A force merge should be called after enabling the refreshing above. - - For Elasticsearch 6.x, the index should be in read-only mode before proceeding with the force merge: - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "settings": { - "index.blocks.write": true - } }' - ``` - - Then, initiate the force merge: - - ```shell - curl --request POST 'localhost:9200/gitlab-production/_forcemerge?max_num_segments=5' - ``` - - After this, if your index is in read-only mode, switch back to read-write: - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "settings": { - "index.blocks.write": false - } }' - ``` - -1. After the indexing has completed, enable [**Search with Elasticsearch enabled**](#enable-advanced-search). - -### Deleted documents - -Whenever a change or deletion is made to an indexed GitLab object (a merge request description is changed, a file is deleted from the default branch in a repository, a project is deleted, etc), a document in the index is deleted. However, since these are "soft" deletes, the overall number of "deleted documents", and therefore wasted space, increases. Elasticsearch does intelligent merging of segments in order to remove these deleted documents. However, depending on the amount and type of activity in your GitLab installation, it's possible to see as much as 50% wasted space in the index. - -In general, we recommend letting Elasticsearch merge and reclaim space automatically, with the default settings. From [Lucene's Handling of Deleted Documents](https://www.elastic.co/blog/lucenes-handling-of-deleted-documents "Lucene's Handling of Deleted Documents"), _"Overall, besides perhaps decreasing the maximum segment size, it is best to leave Lucene's defaults as-is and not fret too much about when deletes are reclaimed."_ - -However, some larger installations may wish to tune the merge policy settings: - -- Consider reducing the `index.merge.policy.max_merged_segment` size from the default 5 GB to maybe 2 GB or 3 GB. Merging only happens when a segment has at least 50% deletions. Smaller segment sizes will allow merging to happen more frequently. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ - --data '{ - "index" : { - "merge.policy.max_merged_segment": "2gb" - } - }' - ``` - -- You can also adjust `index.merge.policy.reclaim_deletes_weight`, which controls how aggressively deletions are targeted. But this can lead to costly merge decisions, so we recommend not changing this unless you understand the tradeoffs. - - ```shell - curl --request PUT localhost:9200/gitlab-production/_settings ---header 'Content-Type: application/json' \ - --data '{ - "index" : { - "merge.policy.reclaim_deletes_weight": "3.0" - } - }' - ``` - -- Do not do a [force merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") to remove deleted documents. A warning in the [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html "Force Merge") states that this can lead to very large segments that may never get reclaimed, and can also cause significant performance or availability issues. - -## Index large instances with dedicated Sidekiq nodes or processes - -Indexing a large instance can be a lengthy and resource-intensive process that has the potential -of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and -availability. - -As GitLab allows you to start multiple Sidekiq processes, you can create an -additional process dedicated to indexing a set of queues (or queue group). This way, you can -ensure that indexing queues always have a dedicated worker, while the rest of the queues have -another dedicated worker to avoid contention. - -For this purpose, use the [queue selector](../administration/operations/extra_sidekiq_processes.md#queue-selector) -option that allows a more general selection of queue groups using a [worker matching query](../administration/operations/extra_sidekiq_routing.md#worker-matching-query). - -To handle these two queue groups, we generally recommend one of the following two options. You can either: - -- [Use two queue groups on one single node](#single-node-two-processes). -- [Use two queue groups, one on each node](#two-nodes-one-process-for-each). - -For the steps below, consider: - -- `feature_category=global_search` as an indexing queue group with its own Sidekiq process. -- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process. - -### Single node, two processes - -To create both an indexing and a non-indexing Sidekiq process in one node: - -1. On your Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=global_search", - "feature_category!=global_search" - ] - ``` - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -WARNING: -When starting multiple processes, the number of processes cannot exceed the number of CPU -cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject -to the available workload and concurrency settings. For more details, see how to -[run multiple Sidekiq processes](../administration/operations/extra_sidekiq_processes.md). - -### Two nodes, one process for each - -To handle these queue groups on two nodes: - -1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=global_search" - ] - ``` - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category!=global_search" - ] - ``` - - to set up a non-indexing Sidekiq process. - -1. Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) -for the changes to take effect. - -## Reverting to Basic Search - -Sometimes there may be issues with your Elasticsearch index data and as such -GitLab allows you to revert to "basic search" when there are no search -results and assuming that basic search is supported in that scope. This "basic -search" behaves as though you don't have Advanced Search enabled at all for -your instance and search using other data sources (such as PostgreSQL data and Git -data). - -## Data recovery: Elasticsearch is a secondary data store only - -The use of Elasticsearch in GitLab is only ever as a secondary data store. -This means that all of the data stored in Elasticsearch can always be derived -again from other data sources, specifically PostgreSQL and Gitaly. Therefore, if -the Elasticsearch data store is ever corrupted for whatever reason, you can reindex everything from scratch. - -## Troubleshooting - -One of the most valuable tools for identifying issues with the Elasticsearch -integration are logs. The most relevant logs for this integration are: - -1. [`sidekiq.log`](../administration/logs.md#sidekiqlog) - All of the - indexing happens in Sidekiq, so much of the relevant logs for the - Elasticsearch integration can be found in this file. -1. [`elasticsearch.log`](../administration/logs.md#elasticsearchlog) - There - are additional logs specific to Elasticsearch that are sent to this file - that may contain useful diagnostic information about searching, - indexing or migrations. - -Here are some common pitfalls and how to overcome them. - -### How can I verify that my GitLab instance is using Elasticsearch? - -There are a couple of ways to achieve that: - -- Whenever you perform a search there is a link on the search results page - in the top right hand corner saying "Advanced search functionality is enabled". - This is always correctly identifying whether the current project/namespace - being searched is using Elasticsearch. - -- From the Admin Area under **Settings > Advanced Search** check that the - Advanced Search settings are checked. - - Those same settings there can be obtained from the Rails console if necessary: - - ```ruby - ::Gitlab::CurrentSettings.elasticsearch_search? # Whether or not searches will use Elasticsearch - ::Gitlab::CurrentSettings.elasticsearch_indexing? # Whether or not content will be indexed in Elasticsearch - ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces - ``` - -- If Elasticsearch is limited to specific namespaces and you need to know if - Elasticsearch is being used for a specific project or namespace, you can use - the Rails console: - - ```ruby - ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Namespace.find_by_full_path("/my-namespace")) - ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) - ``` - -### I updated GitLab and now I can't find anything - -We continuously make updates to our indexing strategies and aim to support -newer versions of Elasticsearch. When indexing changes are made, it may -be necessary for you to [reindex](#zero-downtime-reindexing) after updating GitLab. - -### I indexed all the repositories but I can't get any hits for my search term in the UI - -Make sure you indexed all the database data [as stated above](#enable-advanced-search). - -If there aren't any results (hits) in the UI search, check if you are seeing the same results via the rails console (`sudo gitlab-rails console`): - -```ruby -u = User.find_by_username('your-username') -s = SearchService.new(u, {:search => 'search_term', :scope => 'blobs'}) -pp s.search_objects.to_a -``` - -Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side: - -```shell -curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<search_term> -``` - -More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. - -It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../administration/troubleshooting/elasticsearch.md#search-results-workflow). - -NOTE: -The above instructions are not to be used for scenarios that only index a [subset of namespaces](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). - -See [Elasticsearch Index Scopes](#advanced-search-index-scopes) for more information on searching for specific types of data. - -### I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything - -You must re-run all the Rake tasks to reindex the database, repositories, and wikis. - -### The indexing process is taking a very long time - -The more data present in your GitLab instance, the longer the indexing process takes. - -### There are some projects that weren't indexed, but I don't know which ones - -You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. - -### No new data is added to the Elasticsearch index when I push code - -NOTE: -This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. - -When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could happen that an error during the process causes one or multiple projects to remain locked. To unlock them, run: - -```shell -sudo gitlab-rake gitlab:elastic:clear_locked_projects -``` - -### `Can't specify parent if no parent field has been configured` error - -If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes, you get -exceptions in lots of different cases: - -```plaintext -Elasticsearch::Transport::Transport::Errors::BadRequest([400] { - "error": { - "root_cause": [{ - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }], - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }, - "status": 400 -}): -``` - -This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, -see details in the [update guide](../update/upgrading_from_source.md). - -### `Elasticsearch::Transport::Transport::Errors::BadRequest` - -If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). -There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. - -### `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` - -```plaintext -[413] {"Message":"Request size exceeded 10485760 bytes"} -``` - -This exception is seen when your Elasticsearch cluster is configured to reject requests above a certain size (10MiB in this case). This corresponds to the `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a larger size and restart your Elasticsearch cluster. - -AWS has [fixed limits](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#network-limits) for this setting ("Maximum size of HTTP request payloads"), based on the size of the underlying instance. - -### My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly - -**For a single node Elasticsearch cluster the functional cluster health status is yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. - -WARNING: -Setting the number of replicas to `0` is discouraged (this is not allowed in the GitLab Elasticsearch Integration menu). If you are planning to add more Elasticsearch nodes (for a total of more than 1 Elasticsearch) the number of replicas needs to be set to an integer value larger than `0`. Failure to do so results in lack of redundancy (losing one node corrupts the index). - -If you have a **hard requirement to have a green status for your single node Elasticsearch cluster**, please make sure you understand the risks outlined in the previous paragraph and then run the following query to set the number of replicas to `0`(the cluster no longer tries to create any shard replicas): - -```shell -curl --request PUT localhost:9200/gitlab-production/_settings --header 'Content-Type: application/json' \ - --data '{ - "index" : { - "number_of_replicas" : 0 - } - }' -``` - -### `health check timeout: no Elasticsearch node available` error in Sidekiq - -If you're getting a `health check timeout: no Elasticsearch node available` error in Sidekiq during the indexing process: - -```plaintext -Gitlab::Elastic::Indexer::Error: time="2020-01-23T09:13:00Z" level=fatal msg="health check timeout: no Elasticsearch node available" -``` - -You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticsearch Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be accepted as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a). -After you have corrected the formatting of the URL, delete the index (via the [dedicated Rake task](#gitlab-advanced-search-rake-tasks)) and [reindex the content of your instance](#enable-advanced-search). - -### My Elasticsearch cluster has a plugin and the integration is not working - -Certain 3rd party plugins may introduce bugs in your cluster or for whatever -reason may be incompatible with our integration. You should try disabling -plugins so you can rule out the possibility that the plugin is causing the -problem. - -### Low-level troubleshooting - -There is a [more structured, lower-level troubleshooting document](../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. - -### Elasticsearch `code_analyzer` doesn't account for all code cases - -The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. - -Improvements to the `code_analyzer` pattern and filters are being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621). - -### Some binary files may not be searchable by name - -In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. - -### Last resort to recreate an index - -There may be cases where somehow data never got indexed and it's not in the -queue, or the index is somehow in a state where migrations just cannot -proceed. It is always best to try to troubleshoot the root cause of the problem -using the above [troubleshooting](#troubleshooting) steps. - -If there are no other options, then you always have the option of recreating the -entire index from scratch. If you have a small GitLab installation, this can -sometimes be a quick way to resolve a problem, but if you have a large GitLab -installation, then this might take a very long time to complete. Until the -index is fully recreated, your index does not serve correct search results, -so you may want to disable **Search with Elasticsearch** while it is running. - -If you are sure you've read the above caveats and want to proceed, then you -should run the following Rake task to recreate the entire index from scratch: - -**For Omnibus installations** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -sudo gitlab-rake gitlab:elastic:index -``` - -**For installations from source** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:elastic:index -``` - -### How does Advanced Search handle private projects? - -Advanced Search stores all the projects in the same Elasticsearch indexes, -however, searches only surface results that can be viewed by the user. -Advanced Search honors all permission checks in the application by -filtering out projects that a user does not have access to at search time. - -### Indexing fails with `error: elastic: Error 429 (Too Many Requests)` - -If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: - -- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. - -### Indexing is very slow or fails with `rejected execution of coordinating operation` messages - -Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. -Ensure that your Elasticsearch cluster meets the [system requirements](#system-requirements) and has enough resources -to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). - -### Access requirements for the self-managed AWS OpenSearch Service - -To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies -to contain the actions below. -See [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details. - -```plaintext -es:ESHttpDelete -es:ESHttpGet -es:ESHttpHead -es:ESHttpPost -es:ESHttpPut -es:ESHttpPatch -``` +<!-- This redirect file can be deleted after <2022-09-13>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index c73a8d5e461..99ebce32936 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -108,6 +108,6 @@ Facebook. Facebook generates an app ID and secret key for you to use. installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Facebook icon below the regular sign -in form. Click the icon to begin the authentication process. Facebook asks the +in form. Select the icon to begin the authentication process. Facebook asks the user to sign in and authorize the GitLab application. If everything goes well the user is returned to GitLab and signed in. diff --git a/doc/integration/gitpod.md b/doc/integration/gitpod.md index 84294a99ef9..271505d3d6f 100644 --- a/doc/integration/gitpod.md +++ b/doc/integration/gitpod.md @@ -62,7 +62,7 @@ You can launch Gitpod directly from GitLab in one of these ways: - *From your project's page:* 1. Go to your project, then go to the page you want to edit. - 1. Select the caret **(angle-down)** next to **Web IDE**, and select **Gitpod** + 1. Select the caret (**{chevron-lg-down}**) next to **Web IDE**, and select **Gitpod** from the list:  diff --git a/doc/integration/google.md b/doc/integration/google.md index 596700822cd..4862b898ed5 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -50,7 +50,7 @@ In Google's side: To do so you should: 1. Go to the [Google API Console](https://console.developers.google.com/apis/dashboard). - 1. Click on **ENABLE APIS AND SERVICES** button at the top of the page. + 1. Select **ENABLE APIS AND SERVICES** at the top of the page. 1. Find each of the above APIs. On the page for the API, press the **ENABLE** button. It may take a few minutes for the API to be fully functional. @@ -123,6 +123,6 @@ On your GitLab server: respectively. On the sign in page there should now be a Google icon below the regular sign in -form. Click the icon to begin the authentication process. Google asks the +form. Select the icon to begin the authentication process. Google asks the user to sign in and authorize the GitLab application. If everything goes well the user is returned to GitLab and is signed in. diff --git a/doc/integration/index.md b/doc/integration/index.md index f1d16dc409d..85ebac5b40c 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -60,7 +60,7 @@ GitLab can be integrated with the following enhancements: or [Kroki](../administration/integration/kroki.md) to use diagrams in AsciiDoc and Markdown documents. - Attach merge requests to [Trello](trello_power_up.md) cards. - Enable integrated code intelligence powered by [Sourcegraph](sourcegraph.md). -- Add [Elasticsearch](elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md). +- Add [Elasticsearch](advanced_search/elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md). ## Integrations @@ -104,3 +104,13 @@ After that restart GitLab with: ```shell sudo gitlab-ctl restart ``` + +### Search Sidekiq logs in Kibana + +To locate a specific integration in Kibana, use the following KQL search string: + +```plaintext +`json.integration_class.keyword : "Integrations::Jira" and json.project_path : "path/to/project"` +``` + +You can find information in `json.exception.backtrace`, `json.exception.class`, `json.exception.message`, and `json.message`. diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 7b02580dac1..5b779f22bd3 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -179,7 +179,7 @@ If you get this error message while configuring GitLab, the following are possib - The Jenkins instance is at a local address and is not included in the [GitLab installation's allowlist](../security/webhooks.md#allowlist-for-local-requests). - The credentials for the Jenkins instance do not have sufficient access or are invalid. -- The **Enable authentication for ‘/project’ end-point** checkbox is not selected in your [Jenkin's plugin configuration](#configure-the-jenkins-server). +- The **Enable authentication for `/project` end-point** checkbox is not selected in your [Jenkin's plugin configuration](#configure-the-jenkins-server). ### Error in merge requests - "Could not connect to the CI server" diff --git a/doc/integration/jira/dvcs.md b/doc/integration/jira/dvcs.md index e69b7675a59..d35c21f6187 100644 --- a/doc/integration/jira/dvcs.md +++ b/doc/integration/jira/dvcs.md @@ -69,17 +69,14 @@ you can still perform multiple actions in a single commit. For example: ## Configure a GitLab application for DVCS -We recommend you create and use a `jira` user in GitLab, and use the account +For projects in a single group we recommend you create a [group application](../oauth_provider.md#group-owned-applications). +For projects across multiple groups we recommend you create and use a `jira` user in GitLab, and use the account only for integration work. A separate account ensures regular account -maintenance does not affect your integration. If a `jira` user is not feasible, -you can set up this integration with your own account instead. - -1. In GitLab, [create a user](../../user/profile/account/create_accounts.md) for Jira to - use to connect to GitLab. This user must be added to each project you want Jira to have access to, - or be an administrator to access all projects. -1. Sign in as the `jira` user. -1. On the top bar, in the top right corner, select the user's avatar, and select **Edit profile**. -1. On the left sidebar, select **Applications**. +maintenance does not affect your integration. If a `jira` user or group application is not feasible, +you can set up this integration as an [instance-wide application](../oauth_provider.md#instance-wide-applications) +or with a [user owned application](../oauth_provider.md#user-owned-applications) instead. + +1. Navigate to the [appropriate **Applications** section](../oauth_provider.md#introduction-to-oauth). 1. In the **Name** field, enter a descriptive name for the integration, such as `Jira`. 1. In the **Redirect URI** field, enter the URI appropriate for your version of GitLab, replacing `<gitlab.example.com>` with your GitLab instance domain: @@ -275,7 +272,7 @@ Failed to execute request [https://gitlab.com/api/v4/projects/:id/merge_requests {"message":"403 Forbidden"} ``` -If you find a `{"message":"403 Forbidden"}` error, it is possible that this specific project has some [GitLab features disabled](../../user/project/settings/index.md#sharing-and-permissions). +If you find a `{"message":"403 Forbidden"}` error, it is possible that this specific project has some [GitLab features disabled](../../user/project/settings/index.md#configure-project-visibility-features-and-permissions). In the example above, the merge requests feature is disabled. To resolve the issue, enable the relevant feature: diff --git a/doc/integration/jira/index.md b/doc/integration/jira/index.md index 371f3a4ab8e..4e8b5ca8927 100644 --- a/doc/integration/jira/index.md +++ b/doc/integration/jira/index.md @@ -67,10 +67,13 @@ The authentication method in Jira depends on whether you host Jira on your own s ## Privacy considerations -If you integrate a private GitLab project with Jira using the [**Jira integration**](#jira-integration), -actions in GitLab issues and merge requests linked to a Jira issue leak information -about the private project to non-administrator Jira users. If your installation uses Jira Cloud, -you can use the [GitLab.com for Jira Cloud app](connect-app.md) to avoid this risk. +All Jira integrations share data with Jira to make it visible outside of GitLab. +If you integrate a private GitLab project with Jira, the private data is +shared with users who have access to your Jira project. + +The [**Jira project integration**](#jira-integration) posts GitLab data in the form of comments in Jira issues. +The GitLab.com for Jira Cloud app and Jira DVCS connector share this data through the [**Jira Development Panel**](development_panel.md). +This method provides more fine-grained access control because access can be restricted to certain user groups or roles. ## Third-party Jira integrations @@ -91,6 +94,8 @@ set up for the integration has permission to: Jira issue references and update comments do not work if the GitLab issue tracker is disabled. +If you [restrict IP addresses for Jira access](https://support.atlassian.com/security-and-access-policies/docs/specify-ip-addresses-for-product-access/), make sure you add your self-managed IP addresses or [GitLab.com IP range](../../user/gitlab_com/index.md#ip-range) to the allowlist in Jira. + ### GitLab cannot close a Jira issue If GitLab cannot close a Jira issue: @@ -111,3 +116,9 @@ authenticate with the Jira site. To fix this error, sign in to your Jira instance and complete the CAPTCHA. + +### Jira integration does not work for imported project + +There is a [known bug](https://gitlab.com/gitlab-org/gitlab/-/issues/341571) +where the Jira integration sometimes does not work for a project that has been imported. +As a workaround, disable the integration and then re-enable it. diff --git a/doc/integration/jira/issues.md b/doc/integration/jira/issues.md index aaff5de767b..2d9e928e654 100644 --- a/doc/integration/jira/issues.md +++ b/doc/integration/jira/issues.md @@ -54,10 +54,11 @@ You can [disable comments](#disable-comments-on-jira-issues) on issues. You can prevent merge requests from being merged if they do not refer to a Jira issue. To enforce this: -1. Navigate to your project's **Settings > General** page. -1. Expand the **Merge requests** section. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Merge requests**. 1. Under **Merge checks**, select the **Require an associated issue from Jira** checkbox. -1. Select **Save** for the changes to take effect. +1. Select **Save**. After you enable this feature, a merge request that doesn't reference an associated Jira issue can't be merged. The merge request displays the message diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index 73674b66f05..78c92631e4f 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -354,7 +354,7 @@ When using Kerberos ticket-based authentication in an Active Directory domain, it may be necessary to increase the maximum header size allowed by NGINX, as extensions to the Kerberos protocol may result in HTTP authentication headers larger than the default size of 8kB. Configure `large_client_header_buffers` -to a larger value in [the NGINX configuration](http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers). +to a larger value in [the NGINX configuration](https://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers). ## Troubleshooting @@ -386,7 +386,7 @@ above error message. To fix this, ensure that the forward and reverse DNS for your GitLab server match. So for instance, if you access GitLab as `gitlab.example.com`, resolving -to IP address `1.2.3.4`, then `4.3.2.1.in-addr.arpa` must be a PTR record for +to IP address `1.2.3.4`, then `4.3.2.1.in-addr.arpa` must be a `PTR` record for `gitlab.example.com`. Finally, it's possible that the browser or client machine lack Kerberos support @@ -412,5 +412,5 @@ See also: [Git v2.11 release notes](https://github.com/git/git/blob/master/Docum ## Helpful links - <https://help.ubuntu.com/community/Kerberos> -- <http://blog.manula.org/2012/04/setting-up-kerberos-server-with-debian.html> +- <https://blog.manula.org/2012/04/setting-up-kerberos-server-with-debian.html> - <https://www.roguelynn.com/words/explain-like-im-5-kerberos/> diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index 5ea723abba9..1a60ca3a5fe 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -240,7 +240,7 @@ cd /opt/gitlab/embedded/service/mattermost sudo /opt/gitlab/embedded/bin/chpst -e /opt/gitlab/etc/mattermost/env -P -U mattermost:mattermost -u mattermost:mattermost /opt/gitlab/embedded/bin/mattermost --config=/var/opt/gitlab/mattermost/config.json version ``` -Until [#4745](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4745) has been implemented, the command requires quite of bit typing and is hard to remember, so let's make a bash/zsh alias to make it a bit easier to remember. Add the following to your `~/.bashrc` or `~/.zshrc` file: +Until [#4745](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4745) has been implemented, the command requires quite of bit typing and is hard to remember, so let's make a bash or Zsh alias to make it a bit easier to remember. Add the following to your `~/.bashrc` or `~/.zshrc` file: ```shell alias mattermost-cli="cd /opt/gitlab/embedded/service/mattermost && sudo /opt/gitlab/embedded/bin/chpst -e /opt/gitlab/etc/mattermost/env -P -U mattermost:mattermost -u mattermost:mattermost /opt/gitlab/embedded/bin/mattermost --config=/var/opt/gitlab/mattermost/config.json $1" diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 296de9e511a..aac2820a69e 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -293,16 +293,18 @@ When authenticating using LDAP, the user's name and email are always synced. > Introduced in GitLab 12.3. -With certain OmniAuth providers, users can sign in without -using two-factor authentication. +With certain OmniAuth providers, users can sign in without using two-factor authentication (2FA). -To bypass two-factor authentication, you can either: +Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/196131) users must +[set up 2FA](../user/profile/account/two_factor_authentication.md#enable-two-factor-authentication) on their GitLab +account to bypass 2FA. Otherwise, they are prompted to set up 2FA when they sign in to GitLab. + +To bypass 2FA, you can either: - Define the allowed providers using an array (for example, `['twitter', 'google_oauth2']`). - Specify `true` to allow all providers, or `false` to allow none. -This option should be configured only for providers that already have -two-factor authentication. The default is `false`. +This option should be configured only for providers that already have 2FA. The default is `false`. This configuration doesn't apply to SAML. diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 67c24415226..5300018e888 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -15,7 +15,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create 1. Sign in to [Salesforce](https://login.salesforce.com/). -1. In Setup, enter `App Manager` in the Quick Find box, click **App Manager**, then click **New Connected App**. +1. In Setup, enter `App Manager` in the Quick Find box, select **App Manager**, then select **New Connected App**. 1. Fill in the application details into the following fields: - **Connected App Name** and **API Name**: Set to any value but consider something like `<Organization>'s GitLab`, `<Your Name>'s GitLab`, or something else that is descriptive. @@ -24,14 +24,14 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create  -1. Select **API (Enable OAuth Settings)** and click on **Enable OAuth Settings**. +1. Select **API (Enable OAuth Settings)** and select **Enable OAuth Settings**. 1. Fill in the application details into the following fields: - **Callback URL**: The callback URL of your GitLab installation. For example, `https://gitlab.example.com/users/auth/salesforce/callback`. - **Selected OAuth Scopes**: Move `Access your basic information (id, profile, email, address, phone)` and `Allow access to your unique identifier (openid)` to the right column.  -1. Click **Save**. +1. Select **Save**. 1. On your GitLab server, open the configuration file. @@ -86,7 +86,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be a Salesforce icon below the regular sign in form. -Click the icon to begin the authentication process. Salesforce asks the user to sign in and authorize the GitLab application. +Select the icon to begin the authentication process. Salesforce asks the user to sign in and authorize the GitLab application. If everything goes well, the user is returned to GitLab and is signed in. NOTE: diff --git a/doc/integration/saml.md b/doc/integration/saml.md index ee4c34bb924..9f707ba9bc6 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -160,7 +160,7 @@ At a minimum the IdP *must* provide a claim containing the user's email address See [the assertions list](#assertions) for other available claims. On the sign in page there should now be a SAML button below the regular sign in form. -Click the icon to begin the authentication process. If everything goes well the user +Select the icon to begin the authentication process. If everything goes well the user is returned to GitLab and signed in. ### Use multiple SAML identity providers @@ -185,7 +185,7 @@ gitlab_rails['omniauth_providers'] = [ name: 'saml_1', args: { name: 'saml_1', # This is mandatory and must match the provider name - strategy_class: 'OmniAuth::Strategies::SAML' + strategy_class: 'OmniAuth::Strategies::SAML', assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_1/callback', # URL must match the name of the provider ... # Put here all the required arguments similar to a single provider }, @@ -195,7 +195,7 @@ gitlab_rails['omniauth_providers'] = [ name: 'saml_2', args: { name: 'saml_2', # This is mandatory and must match the provider name - strategy_class: 'OmniAuth::Strategies::SAML' + strategy_class: 'OmniAuth::Strategies::SAML', assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml_2/callback', # URL must match the name of the provider ... # Put here all the required arguments similar to a single provider }, @@ -407,6 +407,10 @@ The requirements are the same as the previous settings: } } ``` +## Group Sync + +For information on automatically managing GitLab group membership, see [SAML Group Sync](../user/group/saml_sso/group_sync.md). + ## Bypass two factor authentication If you want some SAML authentication methods to count as 2FA on a per session @@ -486,7 +490,7 @@ In addition to the changes in GitLab, make sure that your IdP is returning the ### `auto_sign_in_with_provider` You can add this setting to your GitLab configuration to automatically redirect you -to your SAML server for authentication. This removes the requirement to click a button +to your SAML server for authentication. This removes the requirement to select a button before actually signing in. For Omnibus package: @@ -791,7 +795,6 @@ Examples: - [ADFS (Active Directory Federation Services)](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) - [Auth0](https://auth0.com/docs/authenticate/protocols/saml/saml-sso-integrations/configure-auth0-saml-identity-provider) -- [PingOne by Ping Identity](http://docs.pingidentity.com/bundle/pingoneforenterprise/page/xsh1564020480660-1.html) GitLab provides the following setup notes for guidance only. If you have any questions on configuring the SAML app, please contact your provider's support. @@ -812,7 +815,7 @@ The following guidance is based on this Okta article, on adding a [SAML Applicat 1. The last part of the configuration is the feedback section where you can just say you're a customer and creating an app for internal use. 1. When you have your app you can see a few tabs on the top of the app's - profile. Click on the SAML 2.0 configuration instructions button. + profile. Select the SAML 2.0 configuration instructions button. 1. On the screen that comes up take note of the **Identity Provider Single Sign-On URL** which you can use for the `idp_sso_target_url` on your GitLab configuration file. diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index a7facf76f05..19eb31ec588 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -88,4 +88,4 @@ Twitter. Twitter generates a client ID and secret key for you to use. 1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. -On the sign in page there should now be a Twitter icon below the regular sign in form. Click the icon to begin the authentication process. Twitter asks the user to sign in and authorize the GitLab application. If everything goes well the user is returned to GitLab and signed in. +On the sign in page there should now be a Twitter icon below the regular sign in form. Select the icon to begin the authentication process. Twitter asks the user to sign in and authorize the GitLab application. If everything goes well the user is returned to GitLab and signed in. diff --git a/doc/integration/vault.md b/doc/integration/vault.md index 9e738f8493d..f85c71a5bb3 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -98,11 +98,11 @@ The following assumes you already have Vault installed and running. 1. Go to your Vault UI (example: [http://127.0.0.1:8200/ui/vault/auth?with=oidc](http://127.0.0.1:8200/ui/vault/auth?with=oidc)). 1. If the `OIDC` method is not currently selected, open the dropdown and select it. - 1. Click the **Sign in With GitLab** button, which opens a modal window: + 1. Select **Sign in With GitLab**, which opens a modal window:  - 1. Click **Authorize** on the modal to allow Vault to sign in through GitLab. This redirects you back to your Vault UI as a signed-in user. + 1. Select **Authorize** to allow Vault to sign in through GitLab. This redirects you back to your Vault UI as a signed-in user.  @@ -127,7 +127,7 @@ The following assumes you already have Vault installed and running. [Redirect URIs](https://www.vaultproject.io/docs/auth/jwt#redirect-uris). After running the command, it presents a link in the terminal. - Click the link in the terminal and a browser tab opens that confirms you're signed into Vault via OIDC: + Select the link in the terminal and a browser tab opens that confirms you're signed into Vault via OIDC:  diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index 4b503af2cf7..71248adc777 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -403,7 +403,7 @@ docker run \ > - Showing related feature flags in issues [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220333) in GitLab 14.1. You can link related issues to a feature flag. In the Feature Flag **Linked issues** section, -click the `+` button and input the issue reference number or the full URL of the issue. +select the `+` button and input the issue reference number or the full URL of the issue. The issues then appear in the related feature flag and the other way round. This feature is similar to the [linked issues](../user/project/issues/related_issues.md) feature. diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md index 008e41f5d64..af42571f82f 100644 --- a/doc/operations/incident_management/alerts.md +++ b/doc/operations/incident_management/alerts.md @@ -205,7 +205,7 @@ To assign an alert:  1. If the right sidebar is not expanded, select - **Expand sidebar** (**{angle-double-right}**) to expand it. + **Expand sidebar** (**{chevron-double-lg-right}**) to expand it. 1. On the right sidebar, locate the **Assignee**, and then select **Edit**. From the list, select each user you want to assign to the alert. diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index 66d04fce14c..e48127b3415 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -58,7 +58,7 @@ GitLab to create incident automatically whenever an alert is triggered: with the Developer role, select **Send a separate email notification to Developers**. Email notifications are also sent to users with the **Maintainer** and **Owner** roles. -1. Click **Save changes**. +1. Select **Save changes**. ### Create incidents via the PagerDuty webhook @@ -149,7 +149,7 @@ displays it in the Incident Details view: For live examples of GitLab incidents, visit the `tanuki-inc` project's [incident list page](https://gitlab.com/gitlab-examples/ops/incident-setup/everyone/tanuki-inc/-/incidents). -Click any incident in the list to display its incident details page. +Select any incident in the list to display its incident details page. ### Summary @@ -180,7 +180,7 @@ charts in the **Metrics** tab:  -When you upload an image, you can associate the image with text or a link to the original graph. +When you upload an image, you can associate the image with text or a link to the original graph.  @@ -199,7 +199,7 @@ field populated. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227836) in GitLab 13.5. -To quickly see the latest updates on an incident, click +To quickly see the latest updates on an incident, select **{history}** **Turn recent updates view on** in the comment bar to display comments un-threaded and ordered chronologically, newest to oldest: @@ -217,11 +217,11 @@ every 15 minutes so you do not have to refresh the page to see the time remainin To configure the timer: 1. Navigate to **Settings > Monitor**. -1. Scroll to **Incidents** and click **Expand**, then select the +1. Scroll to **Incidents** and select **Expand**, then select the **Incident settings** tab. 1. Select **Activate "time to SLA" countdown timer**. 1. Set a time limit in increments of 15 minutes. -1. Click **Save changes**. +1. Select **Save changes**. After you enable the SLA countdown timer, the **Time to SLA** attribute is displayed as a column in the Incidents List, and as a field on newly created Incidents. If @@ -250,18 +250,14 @@ You can also change the severity using the [`/severity` quick action](../../user ### Add a to-do item -Add a to-do for incidents that you want to track in your to-do list. Click the -**Add a to do** button at the top of the right-hand side bar to add a to-do item. +Add a to-do for incidents that you want to track in your to-do list. Select +**Add a to do** at the top of the right-hand side bar to add a to-do item. ### Change incident status > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. - -FLAG: -This feature is available by default. To disable it per project or for your entire -instance, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) -named `incident_escalations`. +> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. For users with the Developer role or higher, select **Edit** in the **Status** section of the right-hand side bar of an incident, then select a status. **Triggered** is the default status for @@ -281,11 +277,7 @@ updating the incident status also updates the alert status. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. - -FLAG: -This feature is available by default. To disable it per project or for your entire -instance, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) -named `incident_escalations`. +> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. For users with the Developer role or higher, select **Edit** in the **Escalation policy** section of the right-hand side bar of an incident, then select a policy. By default, new incidents do not have @@ -336,7 +328,7 @@ With at least the Maintainer role, you can enable 1. Navigate to **Settings > Monitor > Incidents** and expand **Incidents**. 1. Check the **Automatically close associated Incident** checkbox. -1. Click **Save changes**. +1. Select **Save changes**. When GitLab receives a **Recovery Alert**, it closes the associated incident. This action is recorded as a system message on the incident indicating that it diff --git a/doc/operations/incident_management/integrations.md b/doc/operations/incident_management/integrations.md index fe05cb9fda0..6e65fe875c5 100644 --- a/doc/operations/incident_management/integrations.md +++ b/doc/operations/incident_management/integrations.md @@ -10,9 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) from GitLab Ultimate to GitLab Free in 12.8. GitLab can accept alerts from any source via a webhook receiver. This can be configured -generically or, in GitLab versions 13.1 and greater, you can configure -[External Prometheus instances](../metrics/alerts.md#external-prometheus-instances) -to use this endpoint. +generically. ## Integrations list @@ -27,8 +25,7 @@ the integration name, type, and status (enabled or disabled): ## Configuration -GitLab can receive alerts via a HTTP endpoint that you configure, -or the [Prometheus integration](#external-prometheus-integration). +GitLab can receive alerts via a HTTP endpoint that you configure. ### Single HTTP Endpoint @@ -59,19 +56,19 @@ and you can [customize the payload](#customize-the-alert-payload-outside-of-gitl 1. Expand the **Alerts** section. 1. For each endpoint you want to create: - 1. Click the **Add new integration** button. + 1. Select **Add new integration**. 1. In the **Select integration type** dropdown menu, select **HTTP Endpoint**. 1. Name the integration. 1. Toggle the **Active** alert setting. The **URL** and **Authorization Key** for the webhook configuration are available in the **View credentials** tab after you save the integration. You must also input the URL and Authorization Key in your external service. 1. Optional. To map fields from your monitoring tool's alert to GitLab fields, enter a sample - payload and click **Parse payload for custom mapping**. Valid JSON is required. If you update + payload and select **Parse payload for custom mapping**. Valid JSON is required. If you update a sample payload, you must also remap the fields. 1. Optional. If you provided a valid sample payload, select each value in **Payload alert key** to [map to a **GitLab alert key**](#map-fields-in-custom-alerts). - 1. To save your integration, click **Save Integration**. If desired, you can send a test alert + 1. To save your integration, select **Save Integration**. If desired, you can send a test alert from your integration's **Send test alert** tab after the integration is created. The new HTTP Endpoint displays in the [integrations list](#integrations-list). @@ -89,12 +86,6 @@ GitLab fields when you [create an HTTP endpoint](#http-endpoints):  -### External Prometheus integration - -For GitLab versions 13.1 and greater, read -[External Prometheus Instances](../metrics/alerts.md#external-prometheus-instances) -to configure alerts for this integration. - ## Customize the alert payload outside of GitLab For HTTP Endpoints without [custom mappings](#map-fields-in-custom-alerts), you can customize the payload by sending the following @@ -227,11 +218,11 @@ alert to confirm your integration works properly. 1. Sign in as a user with at least the Developer role. 1. Navigate to **Settings > Monitor** in your project. -1. Click **Alerts** to expand the section. -1. Click the **{settings}** settings icon on the right side of the integration in [the list](#integrations-list). +1. Select **Alerts** to expand the section. +1. Select the **{settings}** settings icon on the right side of the integration in [the list](#integrations-list). 1. Select the **Send test alert** tab to open it. 1. Enter a test payload in the payload field (valid JSON is required). -1. Click **Send**. +1. Select **Send**. GitLab displays an error or success message, depending on the outcome of your test. diff --git a/doc/operations/incident_management/paging.md b/doc/operations/incident_management/paging.md index 420454d8d8a..936c297e555 100644 --- a/doc/operations/incident_management/paging.md +++ b/doc/operations/incident_management/paging.md @@ -52,11 +52,7 @@ or stop alert escalations by [updating the alert's status](alerts.md#update-an-a > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5716) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `incident_escalations`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) in GitLab 14.10. - -FLAG: -This feature is available by default. To disable it per project or for your entire -instance, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) -named `incident_escalations`. +> - [Feature flag `incident_escalations`](https://gitlab.com/gitlab-org/gitlab/-/issues/345769) removed in GitLab 15.1. For incidents, paging on-call responders is optional for each individual incident. To begin escalating the incident, [set the incident's escalation policy](incidents.md#change-escalation-policy). diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md index 0e446f04b9b..12bd975db5d 100644 --- a/doc/operations/metrics/alerts.md +++ b/doc/operations/metrics/alerts.md @@ -17,46 +17,15 @@ your team when environment performance falls outside of the boundaries you set. Alerts are not currently supported for [Prometheus cluster integrations](../../user/clusters/integrations.md). -## External Prometheus instances +<!--- start_remove The following content will be removed on remove_date: '2022-09-22' --> -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/42640) to GitLab Free in 12.10. +## External Prometheus instances (removed) + +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219142) in GitLab 13.2 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/338834) in 15.0. +To manually configure a Prometheus server, we recommend +you use the [generic alerts integration](../incident_management/integrations.md). -For manually configured Prometheus servers, GitLab provides a notify endpoint for -use with Prometheus webhooks. If you have manual configuration enabled, an -**Alerts** section is added to **Settings > Integrations > Prometheus**. -This section contains the needed **URL** and **Authorization Key**. The -**Reset Key** button invalidates the key and generates a new one. - - - -To send GitLab alert notifications, copy the **URL** and **Authorization Key** into the -[`webhook_configs`](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) -section of your Prometheus Alertmanager configuration: - -```yaml -receivers: - - name: gitlab - webhook_configs: - - http_config: - authorization: - type: Bearer - credentials: 9e1cbfcd546896a9ea8be557caf13a76 - send_resolved: true - url: http://192.168.178.31:3001/root/manual_prometheus/prometheus/alerts/notify.json - # Rest of configuration omitted - # ... -``` - -For GitLab to associate your alerts with an [environment](../../ci/environments/index.md), -you must configure a `gitlab_environment_name` label on the alerts you set up in -Prometheus. The value of this should match the name of your environment in GitLab. - -You can display alerts with a `gitlab_environment_name` of `production` -[on a dashboard](../../user/operations_dashboard/index.md#adding-a-project-to-the-dashboard). - -In GitLab versions 13.1 and greater, you can configure your manually configured -Prometheus server to use the -[Generic alerts integration](../incident_management/integrations.md). +<!--- end_remove --> ## Trigger actions from alerts **(ULTIMATE)** @@ -67,7 +36,7 @@ Alerts can be used to trigger actions, like opening an issue automatically 1. Enable the option to create issues. 1. Choose the [issue template](../../user/project/description_templates.md) to create the issue from. 1. Optionally, select whether to send an email notification to the developers of the project. -1. Click **Save changes**. +1. Select **Save changes**. After enabling, GitLab automatically opens an issue when an alert is triggered containing values extracted from the [`alerts` field in webhook payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): diff --git a/doc/operations/metrics/dashboards/develop.md b/doc/operations/metrics/dashboards/develop.md index 513e5532bee..e09f56e27cf 100644 --- a/doc/operations/metrics/dashboards/develop.md +++ b/doc/operations/metrics/dashboards/develop.md @@ -23,14 +23,14 @@ Navigate to the browser-based editor of your choice: - In the **Repository view**: 1. Navigate to **{doc-text}** **Repository > Files**. - 1. Click **{plus}** **Add to tree** and select **New file**, - then click **Select a template type** to see a list of available templates: + 1. Select **{plus}** **Add to tree** and select **New file**, + then select **Select a template type** to see a list of available templates:  - In the **[Web IDE](../../../user/project/web_ide/index.md)**: - 1. Click **Web IDE** when viewing your repository. - 1. Click **{doc-new}** **New file**, then click **Choose a template** to see a list of available templates: + 1. Select **Web IDE** when viewing your repository. + 1. Select **{doc-new}** **New file**, then select **Choose a template** to see a list of available templates:  ## Custom dashboard templates **(PREMIUM SELF)** diff --git a/doc/operations/metrics/dashboards/index.md b/doc/operations/metrics/dashboards/index.md index 052f678f1b7..152e3dca27d 100644 --- a/doc/operations/metrics/dashboards/index.md +++ b/doc/operations/metrics/dashboards/index.md @@ -33,10 +33,10 @@ To create a new dashboard from the GitLab user interface: 1. Sign in to GitLab as a user with Maintainer or Owner [permissions](../../../user/permissions.md#project-members-permissions). 1. Navigate to your dashboard at **Monitor > Metrics**. -1. In the top-right corner of your dashboard, click the **{ellipsis_v}** **More actions** menu, +1. In the top-right corner of your dashboard, select the **{ellipsis_v}** **More actions** menu, and select **Create new**:  -1. In the modal window, click **Open Repository**, then follow the instructions +1. In the modal window, select **Open Repository**, then follow the instructions for creating a new dashboard from the command line. To create a new dashboard from the command line: @@ -84,7 +84,7 @@ with the **Add Panel** page: 1. Sign in to GitLab as a user with Maintainer or Owner [permissions](../../../user/permissions.md#project-members-permissions). -1. Click **Add panel** in the **{ellipsis_v}** **More actions** menu. +1. Select **Add panel** in the **{ellipsis_v}** **More actions** menu. NOTE: You can only add panels to custom dashboards. @@ -92,7 +92,7 @@ with the **Add Panel** page:  1. In the **Define and preview panel** section, paste in the YAML you want to preview in the **Panel YAML** field. -1. Click **Preview panel**, and GitLab displays a preview of the chart below the +1. Select **Preview panel**, and GitLab displays a preview of the chart below the `Define and preview panel` section:  @@ -106,8 +106,8 @@ The resulting `.yml` file can be customized and adapted to your project. You can decide to save the dashboard `.yml` file in the project's **default** branch or in a new branch. To duplicate a GitLab-defined dashboard: -1. Click **Duplicate current dashboard** in the **{ellipsis_v}** **More actions** menu. -1. Enter the filename and other information, such as the new commit's message, and click **Duplicate**. +1. Select **Duplicate current dashboard** in the **{ellipsis_v}** **More actions** menu. +1. Enter the filename and other information, such as the new commit's message, and select **Duplicate**. 1. Select a branch to add your dashboard to: - *If you select your **default** branch,* the new dashboard becomes immediately available. - *If you select another branch,* this branch should be merged to your **default** branch first. @@ -133,7 +133,7 @@ any chart on a dashboard: The options are: - **Expand panel** - Displays a larger version of a visualization. To return to - the dashboard, click the **Back** button in your browser, or press the <kbd>Escape</kbd> key. + the dashboard, select the **Back** button in your browser, or press the <kbd>Escape</kbd> key. ([Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0.) - **View logs** **(ULTIMATE)** - Displays [Logs](../../../user/project/clusters/kubernetes_pod_logs.md), if they are enabled. If used in conjunction with the [timeline zoom](#timeline-zoom-and-url-sharing) @@ -146,7 +146,7 @@ The options are: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198910) in GitLab 12.8. You can use the **Timeline zoom** function at the bottom of a chart to zoom in -on a date and time of your choice. When you click and drag the sliders to select +on a date and time of your choice. When you select and drag the sliders to select a different beginning or end date of data to display, GitLab adds your selected start and end times to the URL, enabling you to share specific time frames more easily. diff --git a/doc/operations/metrics/dashboards/settings.md b/doc/operations/metrics/dashboards/settings.md index aae5a623420..45f0e906e9c 100644 --- a/doc/operations/metrics/dashboards/settings.md +++ b/doc/operations/metrics/dashboards/settings.md @@ -28,12 +28,12 @@ time zone: 1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). 1. Navigate to **Settings > Monitor**. -1. Scroll to **Metrics Dashboard** and click **Expand**. +1. Scroll to **Metrics Dashboard** and select **Expand**. 1. In the **Dashboard timezone** select box, select *User's local timezone* or *UTC*:  -1. Click **Save changes**. +1. Select **Save changes**. ## Link to an external dashboard @@ -44,12 +44,12 @@ existing external dashboards: 1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). 1. Navigate to **Settings > Monitor**. -1. Scroll to **Metrics Dashboard** and click **Expand**. +1. Scroll to **Metrics Dashboard** and select **Expand**. 1. In **External dashboard URL**, provide the URL to your external dashboard:  -1. Click **Save changes**. +1. Select **Save changes**. GitLab displays a **View full dashboard** button in the top right corner of your [monitoring dashboard](../../../ci/environments/index.md#monitor-environments) diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md index 44b998dc3ee..3fd5daba4fb 100644 --- a/doc/operations/metrics/embed.md +++ b/doc/operations/metrics/embed.md @@ -48,8 +48,8 @@ GitLab unfurls the link as an embedded metrics panel:  -You can also embed a single chart. To get a link to a chart, click the -**{ellipsis_v}** **More actions** menu in the upper right corner of the chart, +You can also embed a single chart. To get a link to a chart, select +**{ellipsis_v}** **More actions** in the upper right corner of the chart, and select **Copy link to chart**, as shown in this example:  diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md index 5bfb097619d..700bf821a49 100644 --- a/doc/operations/metrics/embed_grafana.md +++ b/doc/operations/metrics/embed_grafana.md @@ -52,7 +52,7 @@ To set up the Grafana API in Grafana: 1. To enable the integration, check the **Active** checkbox. 1. For **Grafana URL**, enter the base URL of the Grafana instance. 1. For **API Token**, enter the Administrator API token you just generated. -1. Click **Save Changes**. +1. Select **Save Changes**. ### Generate a link to a panel @@ -61,10 +61,10 @@ To generate a link to a panel: 1. In Grafana, go to the dashboard you wish to embed a panel from. 1. In the upper-left corner of the page, select a specific value for each variable required for the queries in the dashboard. -1. In Grafana click on a panel's title, then click **Share** to open the panel's sharing dialog to +1. In Grafana select a panel's title, then select **Share** to open the panel's sharing dialog to the **Link** tab. - If you click the dashboard's share button instead, GitLab attempts to embed the first supported + If you select the dashboard's share button instead, GitLab attempts to embed the first supported panel on the dashboard (if available). 1. If your Prometheus queries use Grafana's custom template variables, ensure the **Template variables** option is set to on. Only the Grafana global template variables diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index 937bd329570..916fe6b0ea9 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -87,7 +87,7 @@ navigation bar contains: [custom dashboards](dashboards/index.md). ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5.) - **Duplicate current dashboard** - Save a [complete copy of a dashboard](dashboards/index.md#duplicate-a-gitlab-defined-dashboard). Only available on GitLab-defined dashboards. - - **Star dashboard** **{star-o}** - Click to mark a dashboard as a favorite. + - **Star dashboard** **{star-o}** - Mark a dashboard as a favorite. Starred dashboards display a solid star **{star}** button, and display first in the **Dashboard** dropdown list. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0.) diff --git a/doc/operations/tracing.md b/doc/operations/tracing.md index e68d7077328..45f2febd73e 100644 --- a/doc/operations/tracing.md +++ b/doc/operations/tracing.md @@ -46,6 +46,6 @@ GitLab provides an easy way to open the Jaeger UI from within your project: 1. [Set up Jaeger](https://www.jaegertracing.io) and configure your application using one of the [client libraries](https://www.jaegertracing.io/docs/latest/client-libraries/). 1. Navigate to your project's **Settings > Monitor** and provide the Jaeger URL. -1. Click **Save changes** for the changes to take effect. +1. Select **Save changes** for the changes to take effect. 1. You can now visit **Monitor > Tracing** in your project's sidebar and GitLab redirects you to the configured Jaeger URL. diff --git a/doc/policy/alpha-beta-support.md b/doc/policy/alpha-beta-support.md index 1e3b591735c..a2612e2bac7 100644 --- a/doc/policy/alpha-beta-support.md +++ b/doc/policy/alpha-beta-support.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index 8ff4e4c5035..e3910115657 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments type: concepts diff --git a/doc/push_rules/push_rules.md b/doc/push_rules/push_rules.md deleted file mode 100644 index ab87cba9da3..00000000000 --- a/doc/push_rules/push_rules.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../user/project/repository/push_rules.md' -remove_date: '2022-05-10' ---- - -This document was moved to [another location](../user/project/repository/push_rules.md). - -<!-- This redirect file can be deleted after <2022-05-10>. --> -<!-- Redirects that point to other docs in the same project expire in three months. --> -<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> -<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 7ff03989c61..d14263f660a 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -304,6 +304,9 @@ sudo -u git -H bundle exec rake gitlab:backup:create SKIP=db,uploads RAILS_ENV=p #### Skipping tar creation +NOTE: +It is not possible to skip the tar creation when using [object storage](#uploading-backups-to-a-remote-cloud-storage) for backups. + The last part of creating a backup is generation of a `.tar` file containing all the parts. In some cases (for example, if the backup is picked up by other backup software) creating a `.tar` file might be wasted effort or even directly @@ -390,7 +393,7 @@ bundle for each repository. There must be an existing backup to create an increm - In GitLab 14.9 and 14.10, use the `BACKUP=<timestamp_of_backup>` option to choose the backup to use. The chosen previous backup is overwritten. - In GitLab 15.0 and later, use the `PREVIOUS_BACKUP=<timestamp_of_backup>` option to choose the backup to use. By default, a backup file is created - as documented in the [Backup timestamp](#backup-timestamp) section. You can override the `[TIMESTAMP]` portion of the filename by setting the + as documented in the [Backup timestamp](#backup-timestamp) section. You can override the `[TIMESTAMP]` portion of the filename by setting the [`BACKUP` environment variable](#backup-filename). To create an incremental backup, run: @@ -426,11 +429,37 @@ For example, for installations from source: sudo -u git -H bundle exec rake gitlab:backup:create REPOSITORIES_STORAGES=storage1,storage2 ``` +#### Back up specific repositories + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88094) in GitLab 15.1. + +You can back up a specific repositories using the `REPOSITORIES_PATHS` option. +The option accepts a comma-separated list of project and group paths. If you +specify a group path, all repositories in all projects in the group and +descendent groups are included. + +For example, to back up all repositories for all projects in **Group A** (`group-a`), and the repository for **Project C** in **Group B** (`group-b/project-c`): + +- Omnibus GitLab installations: + + ```shell + sudo gitlab-backup create REPOSITORIES_PATHS=group-a,group-b/project-c + ``` + +- Installations from source: + + ```shell + sudo -u git -H bundle exec rake gitlab:backup:create REPOSITORIES_PATHS=group-a,group-b/project-c + ``` + #### Uploading backups to a remote (cloud) storage -You can let the backup script upload (using the [Fog library](http://fog.io/)) +NOTE: +It is not possible to [skip the tar creation](#skipping-tar-creation) when using object storage for backups. + +You can let the backup script upload (using the [Fog library](https://fog.io/)) the `.tar` file it creates. In the following example, we use Amazon S3 for -storage, but Fog also lets you use [other storage providers](http://fog.io/storage/). +storage, but Fog also lets you use [other storage providers](https://fog.io/storage/). GitLab also [imports cloud drivers](https://gitlab.com/gitlab-org/gitlab/-/blob/da46c9655962df7d49caef0e2b9f6bbe88462a02/Gemfile#L113) for AWS, Google, OpenStack Swift, Rackspace, and Aliyun. A local driver is [also available](#uploading-to-locally-mounted-shares). @@ -561,7 +590,7 @@ For installations from source: backup: # snip upload: - # Fog storage connection settings, see http://fog.io/storage/ . + # Fog storage connection settings, see https://fog.io/storage/ . connection: provider: AWS region: eu-west-1 @@ -811,7 +840,7 @@ For installations from source: ```yaml backup: upload: - # Fog storage connection settings, see http://fog.io/storage/ . + # Fog storage connection settings, see https://fog.io/storage/ . connection: provider: Local local_root: '/mnt/backups' @@ -986,9 +1015,9 @@ more of the following options: - `BACKUP=timestamp_of_backup`: Required if more than one backup exists. Read what the [backup timestamp is about](#backup-timestamp). -- `force=yes`: Doesn't ask if the authorized_keys file should get regenerated, +- `force=yes`: Doesn't ask if the `authorized_keys` file should get regenerated, and assumes 'yes' for warning about database tables being removed, - enabling the "Write to authorized_keys file" setting, and updating LDAP + enabling the `Write to authorized_keys file` setting, and updating LDAP providers. If you're restoring into directories that are mount points, you must ensure these directories are @@ -1259,6 +1288,30 @@ For example, for installations from source: sudo -u git -H bundle exec rake gitlab:backup:restore BACKUP=timestamp_of_backup REPOSITORIES_STORAGES=storage1,storage2 ``` +#### Restore specific repositories + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88094) in GitLab 15.1. + +You can restore specific repositories using the `REPOSITORIES_PATHS` option. +The option accepts a comma-separated list of project and group paths. If you +specify a group path, all repositories in all projects in the group and +descendent groups are included. The project and group repositories must exist +within the specified backup. + +For example, to restore all repositories for all projects in **Group A** (`group-a`), and the repository for **Project C** in **Group B** (`group-b/project-c`): + +- Omnibus GitLab installations: + + ```shell + sudo gitlab-backup restore BACKUP=timestamp_of_backup REPOSITORIES_PATHS=group-a,group-b/project-c + ``` + +- Installations from source: + + ```shell + sudo -u git -H bundle exec rake gitlab:backup:restore BACKUP=timestamp_of_backup REPOSITORIES_PATHS=group-a,group-b/project-c + ``` + ## Alternative backup strategies If your GitLab instance contains a lot of Git repository data, you may find the @@ -1354,7 +1407,7 @@ There is an **experimental** script that attempts to automate this process in ## Back up and restore for installations using PgBouncer -Do NOT back up or restore GitLab through a PgBouncer connection. These +Do not back up or restore GitLab through a PgBouncer connection. These tasks must [bypass PgBouncer and connect directly to the PostgreSQL primary database node](#bypassing-pgbouncer), or they cause a GitLab outage. @@ -1365,7 +1418,7 @@ following error message is shown: ActiveRecord::StatementInvalid: PG::UndefinedTable ``` -Each time the GitLab backup runs, GitLab will start generating 500 errors and errors about missing +Each time the GitLab backup runs, GitLab starts generating 500 errors and errors about missing tables will [be logged by PostgreSQL](../administration/logs.md#postgresql-logs): ```plaintext @@ -1427,7 +1480,7 @@ WARNING: Avoid uncoordinated data processing by both the new and old servers, where multiple servers could connect concurrently and process the same data. For example, when using [incoming email](../administration/incoming_email.md), if both GitLab instances are -processing email at the same time, then both instances will end up missing some data. +processing email at the same time, then both instances miss some data. This type of problem can occur with other services as well, such as a [non-packaged database](https://docs.gitlab.com/omnibus/settings/database.html#using-a-non-packaged-postgresql-database-management-server), a non-packaged Redis instance, or non-packaged Sidekiq. @@ -1447,7 +1500,7 @@ To prepare the new server: 1. Copy the [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079) from the old server to avoid man-in-the-middle attack warnings. - See [Manually replicate the primary site’s SSH host keys](../administration/geo/replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) for example steps. + See [Manually replicate the primary site's SSH host keys](../administration/geo/replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) for example steps. 1. [Install and configure GitLab](https://about.gitlab.com/install) except [incoming email](../administration/incoming_email.md): 1. Install GitLab. @@ -1591,7 +1644,7 @@ To prepare the new server: 1. Test that the GitLab instance is working as expected. 1. If applicable, re-enable [incoming email](../administration/incoming_email.md) and test it is working as expected. 1. Update your DNS or load balancer to point at the new server. -1. Unblock new CI/CD jobs from starting by removing the custom NGINX config +1. Unblock new CI/CD jobs from starting by removing the custom NGINX configuration you added previously: ```ruby @@ -1670,6 +1723,7 @@ decrypt those columns, preventing access to the following items: - [Project error tracking](../operations/error_tracking.md) - [Runner authentication](../ci/runners/index.md) - [Project mirroring](../user/project/repository/mirror/index.md) +- [Integrations](../user/project/integrations/index.md) - [Web hooks](../user/project/integrations/webhooks.md) In cases like CI/CD variables and runner authentication, you can experience @@ -1844,12 +1898,14 @@ A similar strategy can be employed for the remaining features. By removing the data that can't be decrypted, GitLab can be returned to operation, and the lost data can be manually replaced. -#### Fix project integrations +#### Fix integrations and webhooks -If you've lost your secrets, the [projects' integrations settings pages](../user/project/integrations/index.md) -are probably displaying `500` error messages. +If you've lost your secrets, the [integrations settings pages](../user/project/integrations/index.md) +and [webhooks settings pages](../user/project/integrations/webhooks.md) are probably displaying `500` error messages. -The fix is to truncate the `web_hooks` table: +The fix is to truncate the affected tables (those containing encrypted columns). +This deletes all your configured integrations, webhooks, and related metadata. +You should verify that the secrets are the root cause before deleting any data. 1. Enter the database console: @@ -1877,11 +1933,11 @@ The fix is to truncate the `web_hooks` table: sudo -u git -H bundle exec rails dbconsole -e production --database main ``` -1. Truncate the table: +1. Truncate the following tables: ```sql -- truncate web_hooks table - TRUNCATE web_hooks CASCADE; + TRUNCATE integrations, chat_names, issue_tracker_data, jira_tracker_data, slack_integrations, web_hooks, zentao_tracker_data, web_hook_logs; ``` ### Container Registry push failures after restoring from a backup diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index c2d3c540cbf..59d8046c8db 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 45d51fe9dfa..e3acefb5520 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/raketasks/index.md b/doc/raketasks/index.md index 96c31047d8d..38b57fab8ca 100644 --- a/doc/raketasks/index.md +++ b/doc/raketasks/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -26,7 +26,7 @@ The following Rake tasks are available for use with GitLab: | [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. | | [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. | | [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. | -| [Elasticsearch](../integration/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | +| [Elasticsearch](../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) | Maintain Elasticsearch in a GitLab instance. | | [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. | | [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | | [Geo maintenance](../administration/raketasks/geo.md) | [Geo](../administration/geo/index.md)-related maintenance. | diff --git a/doc/raketasks/list_repos.md b/doc/raketasks/list_repos.md index 57020491197..4b971c3608f 100644 --- a/doc/raketasks/list_repos.md +++ b/doc/raketasks/list_repos.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/raketasks/sidekiq_job_migration.md b/doc/raketasks/sidekiq_job_migration.md index 313c9c7220b..a3bc8b2959a 100644 --- a/doc/raketasks/sidekiq_job_migration.md +++ b/doc/raketasks/sidekiq_job_migration.md @@ -11,7 +11,7 @@ This operation should be very uncommon. We do not recommend it for the vast majo Sidekiq routing rules allow administrators to re-route certain background jobs from their regular queue to an alternative queue. By default, GitLab uses one queue per background job type. GitLab has over 400 background job types, and so correspondingly it has over 400 queues. -Most administrators will not need to change this setting. In some cases with particularly large background job processing workloads, Redis performance may suffer due to the number of queues that GitLab listens to. +Most administrators do not need to change this setting. In some cases with particularly large background job processing workloads, Redis performance may suffer due to the number of queues that GitLab listens to. If the Sidekiq routing rules are changed, administrators need to take care with the migration to avoid losing jobs entirely. The basic migration steps are: diff --git a/doc/raketasks/user_management.md b/doc/raketasks/user_management.md index 09e1b626c0a..b47afef7145 100644 --- a/doc/raketasks/user_management.md +++ b/doc/raketasks/user_management.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/raketasks/web_hooks.md b/doc/raketasks/web_hooks.md index e48a03f6883..44ca4a3f47e 100644 --- a/doc/raketasks/web_hooks.md +++ b/doc/raketasks/web_hooks.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/security/crime_vulnerability.md b/doc/security/crime_vulnerability.md index 8288f7f6a74..7956e692887 100644 --- a/doc/security/crime_vulnerability.md +++ b/doc/security/crime_vulnerability.md @@ -19,7 +19,7 @@ The TLS Protocol CRIME Vulnerability affects systems that use data compression over HTTPS. Your system might be vulnerable to the CRIME vulnerability if you use SSL Compression (for example, Gzip) or SPDY (which optionally uses compression). -GitLab supports both Gzip and [SPDY](http://nginx.org/en/docs/http/ngx_http_spdy_module.html) and mitigates the CRIME +GitLab supports both Gzip and [SPDY](https://nginx.org/en/docs/http/ngx_http_spdy_module.html) and mitigates the CRIME vulnerability by deactivating Gzip when HTTPS is enabled. The sources of the files are here: @@ -58,7 +58,7 @@ vulnerability. ## References -- NGINX ["Module `ngx_http_spdy_module`"](http://nginx.org/en/docs/http/ngx_http_spdy_module.html) +- NGINX ["Module `ngx_http_spdy_module`"](https://nginx.org/en/docs/http/ngx_http_spdy_module.html) - Tenable Network Security, Inc. ["Transport Layer Security (TLS) Protocol CRIME Vulnerability"](https://www.tenable.com/plugins/index.php?view=single&id=62565) - Wikipedia contributors, ["CRIME"](https://en.wikipedia.org/wiki/CRIME) Wikipedia, The Free Encyclopedia diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index ac532ee491a..695a0d52af6 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -32,8 +32,8 @@ similarly mitigated by a rate limit. You can set these rate limits in the Admin Area of your instance: - [Import/Export rate limits](../user/admin_area/settings/import_export_rate_limits.md) -- [Issues rate limits](../user/admin_area/settings/rate_limit_on_issues_creation.md) -- [Notes rate limits](../user/admin_area/settings/rate_limit_on_notes_creation.md) +- [Issue rate limits](../user/admin_area/settings/rate_limit_on_issues_creation.md) +- [Note rate limits](../user/admin_area/settings/rate_limit_on_notes_creation.md) - [Protected paths](../user/admin_area/settings/protected_paths.md) - [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md) - [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md) @@ -42,6 +42,7 @@ You can set these rate limits in the Admin Area of your instance: - [Files API rate limits](../user/admin_area/settings/files_api_rate_limits.md) - [Deprecated API rate limits](../user/admin_area/settings/deprecated_api_rate_limits.md) - [GitLab Pages rate limits](../administration/pages/index.md#rate-limits) +- [Pipeline rate limits](../user/admin_area/settings/rate_limit_on_pipelines_creation.md) You can set these rate limits using the Rails console: @@ -72,11 +73,16 @@ For configuration information, see ### Git operations using SSH -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78373) in GitLab 14.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78373) in GitLab 14.7 [with a flag](../administration/feature_flags.md) named `rate_limit_gitlab_shell`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79419) in GitLab 14.8. -GitLab rate limits Git operations using SSH by user account and project. If a request from a user for a Git operation -on a project exceeds the rate limit, GitLab drops further connection requests from that user for the project. +FLAG: +On self-managed GitLab, by default this feature is available. To disable the feature, ask an administrator to +[disable the feature flag](../administration/feature_flags.md) named `rate_limit_gitlab_shell`. On GitLab.com, this feature +is available. + +GitLab applies rate limits to Git operations that use SSH by user account and project. When the rate limit is exceeded, GitLab rejects +further connection requests from that user for the project. The rate limit applies at the Git command ([plumbing](https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain)) level. Each command has a rate limit of 600 per minute. For example: @@ -86,9 +92,8 @@ Each command has a rate limit of 600 per minute. For example: Because the same commands are shared by `git-upload-pack`, `git pull`, and `git clone`, they share a rate limit. -The requests/minute threshold for this rate limit is not configurable. Self-managed customers can disable this -rate limit by [disabling the feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) -with `Feature.disable(:rate_limit_gitlab_shell)`. +The requests per minute threshold for this rate limit is not configurable. Self-managed customers can disable this +rate limit. ### Repository archives diff --git a/doc/security/ssh_keys_restrictions.md b/doc/security/ssh_keys_restrictions.md index 272d840ef13..eb92694d236 100644 --- a/doc/security/ssh_keys_restrictions.md +++ b/doc/security/ssh_keys_restrictions.md @@ -48,6 +48,25 @@ By default, the GitLab.com and self-managed settings for the - ECDSA_SK SSH keys are allowed (GitLab 14.8 and later). - ED25519_SK SSH keys are allowed (GitLab 14.8 and later). +### Block banned or compromised keys **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24614) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `ssh_banned_key`. Enabled by default. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature per user, +ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `ssh_banned_key`. +On GitLab.com, this feature is available. + +When users attempt to [add a new SSH key](../user/ssh.md#add-an-ssh-key-to-your-gitlab-account) +to GitLab accounts, the key is checked against a list of SSH keys which are known +to be compromised. Users can't add keys from this list to any GitLab account. +This restriction cannot be configured. This restriction exists because the private +keys associated with the key pair are publicly known, and can be used to access +accounts using the key pair. + +If your key is disallowed by this restriction, [generate a new SSH key pair](../user/ssh.md#generate-an-ssh-key-pair) +to use instead. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index aaa6447cb6c..6d88b61dd05 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -105,8 +105,8 @@ the tiers are no longer mentioned in GitLab documentation: - [External groups](../integration/saml.md#external-groups) - [Required groups](../integration/saml.md#required-groups) - Search: - - [Filtering merge requests by approvers](../user/search/index.md#filtering-merge-requests-by-approvers) - - [Filtering merge requests by "approved by"](../user/search/index.md#filtering-merge-requests-by-approved-by) + - [Filtering merge requests by approvers](../user/project/merge_requests/index.md#filter-merge-requests-by-approvers) + - [Filtering merge requests by "approved by"](../user/project/merge_requests/index.md#filter-merge-requests-by-approved-by) - [Advanced Search (Elasticsearch)](../user/search/advanced_search.md) - [Service Desk](../user/project/service_desk.md) - [Storage usage statistics](../user/usage_quotas.md#storage-usage-statistics) @@ -129,7 +129,7 @@ Bronze-level subscribers: - [Group iterations API](../api/group_iterations.md) - Project milestones API: [Get all burndown chart events for a single milestone](../api/milestones.md#get-all-burndown-chart-events-for-a-single-milestone) - [Project iterations API](../api/iterations.md) - - Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/elasticsearch.md) users + - Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/advanced_search/elasticsearch.md) users - Fields in the [Merge requests API](../api/merge_requests.md) for [merge request approvals](../user/project/merge_requests/approvals/index.md) - Fields in the [Protected branches API](../api/protected_branches.md) that specify users or groups allowed to merge - [Merge request approvals API](../api/merge_request_approvals.md) diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index c94d90f61b8..da84cc6211e 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -241,7 +241,7 @@ You can hover your mouse on the **Renew** button to see the date when it will be 1. Review your renewal details and complete the payment process. 1. Select **Confirm purchase**. -Your updated subscription is applied to your namespace on the renewal period start date. +Your updated subscription is applied to your namespace on the renewal period start date. It may take up to one day for the renewal to be processed. An invoice is generated for the renewal and available for viewing or download on the [View invoices](https://customers.gitlab.com/receipts) page. If you have difficulty during the renewal process, contact the [Support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance. @@ -249,14 +249,14 @@ If you have difficulty during the renewal process, contact the [Support team](ht For details on upgrading your subscription tier, see [Upgrade your GitLab SaaS subscription tier](#upgrade-your-gitlab-saas-subscription-tier). -### Automatic renewal +### Automatic subscription renewal When you enable automatic renewal, the subscription automatically renews on the expiration date without a gap in available service. An invoice is generated for the renewal and available for viewing or download on the [View invoices](https://customers.gitlab.com/receipts) page. -#### Enable or disable automatic renewal +#### Enable or disable automatic subscription renewal To view or change automatic subscription renewal (at the same tier as the previous period), log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in), and: @@ -312,8 +312,8 @@ locked. Projects can only be unlocked by purchasing more storage subscription un You can purchase a storage subscription for your personal or group namespace. NOTE: -Storage subscriptions **[renew automatically](#automatic-renewal) each year**. -You can [cancel the subscription](#enable-or-disable-automatic-renewal) to disable the automatic renewal. +Storage subscriptions **[renew automatically](#automatic-subscription-renewal) each year**. +You can [cancel the subscription](#enable-or-disable-automatic-subscription-renewal) to disable the automatic renewal. #### For your personal namespace @@ -321,8 +321,8 @@ You can [cancel the subscription](#enable-or-disable-automatic-renewal) to disab 1. From either your personal homepage or the group's page, go to **Settings > Usage Quotas**. 1. For each locked project, total by how much its **Usage** exceeds the free quota and purchased storage. You must purchase the storage increment that exceeds this total. -1. Click **Purchase more storage** and you are taken to the Customers Portal. -1. Click **Add new subscription**. +1. Select **Purchase more storage** and you are taken to the Customers Portal. +1. Select **Add new subscription**. 1. Scroll to **Purchase add-on subscriptions** and select **Buy storage subscription**. 1. In the **Subscription details** section select the name of the user or group from the dropdown. 1. Enter the desired quantity of storage packs. diff --git a/doc/subscriptions/gitlab_dedicated/index.md b/doc/subscriptions/gitlab_dedicated/index.md index 7c768d1b403..0b3b44d74e1 100644 --- a/doc/subscriptions/gitlab_dedicated/index.md +++ b/doc/subscriptions/gitlab_dedicated/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -72,8 +72,38 @@ Fill in the following form to contact us and learn more about this offering. See https://gitlab.com/gitlab-com/marketing/marketing-operations/-/issues/6238#note_923358643 --> -<script src="//page.gitlab.com/js/forms2/js/forms2.min.js"></script> +<script src="https://page.gitlab.com/js/forms2/js/forms2.min.js"></script> <form id="mktoForm_3226"></form> -<script>MktoForms2.loadForm("//page.gitlab.com", "194-VVC-221", 3226);</script> +<script>MktoForms2.loadForm("https://page.gitlab.com", "194-VVC-221", 3226);</script> +<style> + #mktoForm_3226 { + font-size: .875rem !important; + } + .mktoLabel { + margin-top: 1rem !important; + padding-bottom: .5rem !important; + font-weight: 600; + } + .mktoHtmlText, + #LblPhone, + .mktoTextField, + #commentCapture, + .mktoField, + .mktoButtonRow button { + width: 20rem !important; + } + .mktoHtmlText { + font-size: .875rem; + } + .mktoButtonRow { + margin: 1em 0; + } + .mktoButtonRow span { + margin-left: 0 !important; + } + .mktoButtonRow button { + margin: 1em 0 1.5em !important; + } +</style> <!-- markdownlint-enable --> diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 58af5787f2b..4caf69d6015 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -53,8 +53,8 @@ at each tier for each product, see: ## Find your subscription -The following chart should help you determine your subscription model. Click -on the list item to go to the respective help page. +The following chart should help you determine your subscription model. Select +the list item to go to the respective help page. ```mermaid graph TD @@ -91,7 +91,7 @@ To change your personal details, including name, billing address, and email addr 1. Select **My account > Account details**. 1. Expand the **Personal details** section. 1. Edit your personal details. -1. Click **Save changes**. +1. Select **Save changes**. ### Change your company details @@ -101,7 +101,7 @@ To change your company details, including company name and VAT number: 1. Select **My account > Account details**. 1. Expand the **Company details** section. 1. Edit the company details. -1. Click **Save changes**. +1. Select **Save changes**. ### Change your payment method @@ -117,7 +117,7 @@ To change your payment method: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). 1. Select **My account > Payment methods**. 1. **Edit** an existing payment method's information or **Add new payment method**. -1. Click **Save Changes**. +1. Select **Save Changes**. #### Set a default payment method @@ -127,7 +127,7 @@ method as the default: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). 1. Select **My account > Payment methods**. 1. **Edit** the selected payment method and check the **Make default payment method** checkbox. -1. Click **Save Changes**. +1. Select **Save Changes**. ### Change the linked account @@ -137,8 +137,8 @@ To change the GitLab.com account linked to your Customers Portal account: [Customers Portal](https://customers.gitlab.com/customers/sign_in). 1. In a separate browser tab, go to [GitLab SaaS](https://gitlab.com) and ensure you are not logged in. -1. On the Customers Portal page, click **My account > Account details**. -1. Under **Your GitLab.com account**, click **Change linked account**. +1. On the Customers Portal page, select **My account > Account details**. +1. Under **Your GitLab.com account**, select **Change linked account**. 1. Log in to the [GitLab SaaS](https://gitlab.com) account you want to link to the Customers Portal account. @@ -164,9 +164,9 @@ Only one namespace can be linked to a subscription. To change the password for this customers portal account: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select the **My account** drop-down and click on **Account details**. +1. Select the **My account** drop-down and select **Account details**. 1. Make the required changes to the **Your password** section. -1. Click **Save changes**. +1. Select **Save changes**. ## Community program subscriptions @@ -183,68 +183,63 @@ Find more information on how to apply and renew at ### GitLab for Open Source -For qualifying open source projects, the [GitLab for Open Source](https://about.gitlab.com/solutions/open-source/) program provides -the top GitLab tier, plus 50,000 CI/CD minutes per month. - -You can find more information about the [program requirements](https://about.gitlab.com/solutions/open-source/join/#requirements), -[renewals](https://about.gitlab.com/solutions/open-source/join/#renewals), -and benefits on the [GitLab for Open Source application page](https://about.gitlab.com/solutions/open-source/join/). +For qualifying open source projects, the [GitLab for Open Source Program](https://about.gitlab.com/solutions/open-source/) provides +GitLab Ultimate, plus 50,000 CI/CD minutes per month. For more information, see [program requirements](https://about.gitlab.com/solutions/open-source/join/#requirements), [renewals](https://about.gitlab.com/solutions/open-source/join/#renewals), and [program benefits](https://about.gitlab.com/solutions/open-source/join/). If you have any questions, send an email to `opensource@gitlab.com` for assistance. -#### Verification for Open Source program +#### License requirements for GitLab for Open Source Program members -As part of the [application verification process](https://about.gitlab.com/solutions/open-source/join/), you must upload <b>three screenshots</b>. -These are the three screenshots that are needed to qualify you for the GitLab for Open Source program. +GitLab for Open Source Program benefits apply to an entire GitLab namespace. To qualify for the GitLab for Open Source Program, **all projects in an applicant's namespace** must carry an [OSI-approved license](https://opensource.org/licenses/). -- [OSI-approved license overview](#screenshot-1-license-overview) -- [OSI-approved license file](#screenshot-2-license-file) -- [Publicly visible settings](#screenshot-3-publicly-visible-settings) +To add a license: -##### OSI-approved license +1. On the top bar, select **Menu > Projects** and find your project. +1. On the overview page, select **Add LICENSE**. If the license you want is not available as a license template, manually copy the entire, unaltered [text of your chosen license](https://opensource.org/licenses/alphabetical) into the `LICENSE` file. Note that GitLab defaults to **All rights reserved** if users do not perform this action. -You must apply an [OSI-approved license](https://opensource.org/licenses/) to each project in your group before you can be verified. +Applicants must add the correct license to each project in their respective groups or namespaces When you're sure you're using OSI-approved licenses for your projects, you can take your screenshots. -Add the license to the LICENSE file so that it shows up in the overview section of the project. This allows contributors to see it at a glance. +#### Verification for Open Source Program -It's best to copy and paste the entire license into the file in its original form. GitLab defaults to **All rights reserved** if no license file is mentioned. -You must ensure that you add the correct license to each project within your group. +As part of the [application verification process](https://about.gitlab.com/solutions/open-source/join/), you must upload **three screenshots**: -After you ensure that you are using OSI-approved licenses for your projects, you can take your screenshots. +- [OSI-approved license overview](#screenshot-1-license-overview) +- [OSI-approved license contents](#screenshot-2-license-contents) +- [Publicly visible settings](#screenshot-3-publicly-visible-settings) + +Benefits of the GitLab Open Source Program apply to all projects in a GitLab namespace. All projects in an eligible namespace must meet program requirements. However, if you submit materials for **one project** in your namespace, the open source program team uses that project to verify the contents of the entire namespace you use when applying to the program. ##### Screenshot 1: License overview -On the left sidebar, select **Project information > Details**. Take a screenshot that includes a view of the license you've chosen for your project. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select your project avatar. If you haven't specified an avatar for your project, the avatar displays as a single letter. +1. Take a screenshot of the project overview that clearly displays the license you've chosen for your project.  -##### Screenshot 2: License file +##### Screenshot 2: License contents -Navigate to one of the license files that you uploaded. You can usually find the license file by selecting **Project information > Details** and scanning the page for the license. -Make sure the screenshot includes the title of the license. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Repository** and locate the project's `LICENSE` file. +1. Take a screenshot of the contents of the file. Make sure the screenshot includes the title of the license.  ##### Screenshot 3: Publicly visible settings -The goal of the GitLab for Open Source program is to enable collaboration on open source projects. -As a pre-condition to collaboration, people must be able to view the open source project. -As a result, we ask that all projects under this license are publicly visible. - -Follow these instructions to take a screenshot of the publicly visible settings: +To be eligible for the GitLab Open Source Program, projects must be publicly visible. To check your project's public visibility settings: - 1. Go to your project and select **Settings**. - 1. Expand **Visibility, project features, permissions**. - 1. Set **Project Visibility** to **Public**. - 1. Ensure others can request access by selecting the **Users can request access** checkbox. - 1. Take the screenshot. Include as much of the publicly visible settings as possible. Make sure to include your project's name in the - upper-left of the screenshot. +1. On the top bar, select **Menu > Projects** and find your project. +1. From the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. From the **Project visibility** dropdown list, select **Public**. +1. Select the **Users can request access** checkbox. +1. Take a screenshot of this view. Include as much of the publicly visible settings as possible. Make sure to include your project's name in the upper-left of the screenshot.  NOTE: -From time to time, GitLab allows exceptions. One or two projects within a group can be private if there is a legitimate need for it, for example, -if a project holds sensitive data. Email `opensource@gitlab.com` with details of your use case to request written permission for exceptions. +Exceptions to this public visibility requirement apply in select circumstances (for example, in cases where a project may hold sensitive data). Email `opensource@gitlab.com` with details of your use case to request written permission for exceptions. ### GitLab for Startups diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index ce49b5a9c05..d592c87d2a8 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -148,12 +148,12 @@ This sync job runs daily around 3AM UTC. If the job fails, it is retried up to 1 The daily job provides **only** the following information to the Customers Portal: -- Company name -- Licensee name -- Licensee email - Date - Timestamp - License key + - Company name (encrypted within license key) + - Licensee name (encrypted within license key) + - Licensee email (encrypted within license key) - Historical maximum user count - Billable users count - GitLab version @@ -273,7 +273,7 @@ If you are an administrator, you can export your license usage into a CSV: 1. On the left sidebar, select **Subscription**. 1. In the top right, select **Export license usage file**. -This file contains the information GitLab uses to manually process quarterly reconciliations or renewals. If your instance is firewalled or air-gapped, you must provide GitLab with this information. +This file contains the information GitLab uses to manually process quarterly reconciliations or renewals. If your instance is firewalled or in an offline environment, you must provide GitLab with this information. The **License Usage** CSV includes the following details: @@ -355,10 +355,12 @@ To add seats to a subscription: 1. Review the **Subscription Upgrade Detail**. The system lists the total price for all users on the system and a credit for what you've already paid. You are only be charged for the net change. 1. Select **Confirm Upgrade**. -The following items are emailed to you: +A payment receipt is emailed to you, which you can also access in the Customers Portal under [**View invoices**](https://customers.gitlab.com/receipts). -- A payment receipt. You can also access this information in the Customers Portal under [**View invoices**](https://customers.gitlab.com/receipts). -- An activation code. [Add this code](../../user/admin_area/license.md) to your instance to use it. +If your subscription was activated with an activation code, the additional seats are reflected in +your instance immediately. If you're using a license file, you receive an updated file. +To add the seats, [add the license file](../../user/admin_area/license_file.md) +to your instance. ### Renew a subscription @@ -428,6 +430,10 @@ before this occurs. - To resume functionality, activate a new license. - To fall back to Free features, delete the expired license. +## Activate a license file or key + +If you have a license file or key, you can activate it [in the Admin Area](../../user/admin_area/license_file.md#activate-gitlab-ee-with-a-license-file-or-key). + ## Contact Support Learn more about: diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md new file mode 100644 index 00000000000..653fe11c59d --- /dev/null +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md @@ -0,0 +1,36 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Use Auto DevOps to deploy to EC2 + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216008) in GitLab 13.6. + +To use [Auto DevOps](../index.md) to deploy to EC2: + +1. Define [your AWS credentials as CI/CD variables](../../../ci/cloud_deployment/index.md#authenticate-gitlab-with-aws). +1. In your `.gitlab-ci.yml` file, reference the `Auto-Devops.gitlab-ci.yml` template. +1. Define a job for the `build` stage named `build_artifact`. For example: + + ```yaml + # .gitlab-ci.yml + + include: + - template: Auto-DevOps.gitlab-ci.yml + + variables: + AUTO_DEVOPS_PLATFORM_TARGET: EC2 + + build_artifact: + stage: build + script: + - <your build script goes here> + artifacts: + paths: + - <built artifact> + ``` + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a video walkthrough of this process, view [Auto Deploy to EC2](https://www.youtube.com/watch?v=4B-qSwKnacA). diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md new file mode 100644 index 00000000000..a9b466653bf --- /dev/null +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md @@ -0,0 +1,38 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Use Auto DevOps to deploy to Amazon ECS + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. + +You can choose to target AWS ECS as a deployment platform instead of using Kubernetes. + +To get started on Auto DevOps to AWS ECS, you must add a specific CI/CD variable. +To do so, follow these steps: + +1. In GitLab, on the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Auto DevOps**. +1. Specify which AWS platform to target during the Auto DevOps deployment + by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable with one of the following values: + - `FARGATE` if the service you're targeting must be of launch type FARGATE. + - `ECS` if you're not enforcing any launch type check when deploying to ECS. + +When you trigger a pipeline, if you have Auto DevOps enabled and if you have correctly +[entered AWS credentials as variables](../../../ci/cloud_deployment/index.md#authenticate-gitlab-with-aws), +your application is deployed to AWS ECS. + +If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, +only the deployment to Kubernetes runs. + +WARNING: +Setting the `AUTO_DEVOPS_PLATFORM_TARGET` variable to `ECS` triggers jobs +defined in the [`Jobs/Deploy/ECS.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml). +However, it's not recommended to [include](../../../ci/yaml/index.md#includetemplate) +it on its own. This template is designed to be used with Auto DevOps only. It may change +unexpectedly causing your pipeline to fail if included on its own. Also, the job +names within this template may also change. Do not override these jobs' names in your +own pipeline, as the override stops working when the name changes. diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md new file mode 100644 index 00000000000..aceb1ed8910 --- /dev/null +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md @@ -0,0 +1,331 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Use Auto DevOps to deploy an application to Google Kubernetes Engine **(FREE)** + +In this tutorial, we'll help you to get started with [Auto DevOps](../index.md) +through an example of how to deploy an application to Google Kubernetes Engine (GKE). + +You are using the GitLab native Kubernetes integration, so you don't need +to create a Kubernetes cluster manually using the Google Cloud Platform console. +You are creating and deploying an application that you create from a GitLab template. + +These instructions also work for self-managed GitLab instances. +Ensure your own [runners are configured](../../../ci/runners/index.md) and +[Google OAuth is enabled](../../../integration/google.md). + +To deploy a project to Google Kubernetes Engine, follow the steps below: + +1. [Configure your Google account](#configure-your-google-account) +1. [Create a new project from a template](#create-a-new-project-from-a-template) +1. [Create a Kubernetes cluster from GitLab](#create-a-kubernetes-cluster-from-gitlab) +1. [Install Ingress](#install-ingress) +1. [Configure your base domain](#configure-your-base-domain) +1. [Enable Auto DevOps](#enable-auto-devops-optional) +1. [Deploy the application](#deploy-the-application) + +## Configure your Google account + +Before creating and connecting your Kubernetes cluster to your GitLab project, +you need a [Google Cloud Platform account](https://console.cloud.google.com). +Sign in with an existing Google account, such as the one you use to access Gmail +or Google Drive, or create a new one. + +1. Follow the steps described in the ["Before you begin" section](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) + of the Kubernetes Engine documentation to enable the required APIs and related services. +1. Ensure you've created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) + with Google Cloud Platform. + +NOTE: +Every new Google Cloud Platform (GCP) account receives [$300 in credit](https://console.cloud.google.com/freetrial), +and in partnership with Google, GitLab is able to offer an additional $200 for new +GCP accounts to get started with the GitLab integration with Google Kubernetes Engine. +[Follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) +and apply for credit. + +## Create a new project from a template + +Use a GitLab project template to get started. As the name suggests, +those projects provide a bare-bones application built on some well-known frameworks. + +1. On the top bar in GitLab, select the plus icon (**{plus-square}**), and select + **New project/repository**. +1. Go to the **Create from template** tab, where you can choose a Ruby on + Rails, Spring, or NodeJS Express project. + For this tutorial, use the Ruby on Rails template. + +  + +1. Give your project a name, optionally a description, and make it public so that + you can take advantage of the features available in the + [GitLab Ultimate plan](https://about.gitlab.com/pricing/). + +  + +1. Select **Create project**. + +Now that you've created a project, create the Kubernetes cluster +to deploy this project to. + +## Create a Kubernetes cluster from GitLab + +1. On your project's landing page, select the button **Add Kubernetes cluster**. + +  + +1. On the **Kubernetes clusters** page, select the **Create a new cluster** option from the **Actions** dropdown menu. + +1. On the **Connect a Kubernetes cluster** page, select **Google GKE**. + +1. Connect with your Google account, and select **Allow** to allow access to your + Google account. (This authorization request is only displayed the first time + you connect GitLab with your Google account.) + + After authorizing access, the **Connect a Kubernetes cluster** page + is displayed. + +1. In the **Enter your Kubernetes cluster certificate details** section, provide + details about your cluster: + + - **Kubernetes cluster name** + - **Environment scope** - Leave this field unchanged. + - **Google Cloud Platform project** - Select a project. When you + [configured your Google account](#configure-your-google-account), a project + should have already been created for you. + - **Zone** - The [region/zone](https://cloud.google.com/compute/docs/regions-zones/) to + create the cluster in. + - **Number of nodes** + - **Machine type** - For more information about + [machine types](https://cloud.google.com/compute/docs/machine-types), see Google's documentation. + - **Enable Cloud Run for Anthos** - Select this checkbox to use the + [Cloud Run](../../../user/project/clusters/add_gke_clusters.md#cloud-run-for-anthos), + Istio, and HTTP Load Balancing add-ons for this cluster. + - **GitLab-managed cluster** - Select this checkbox to + [allow GitLab to manage namespace and service accounts](../../../user/project/clusters/gitlab_managed_clusters.md) for this cluster. + +1. Select **Create Kubernetes cluster**. + +After a couple of minutes, the cluster is created. You can also see its +status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). + +## Install Ingress + +After your cluster is running, you must install NGINX Ingress Controller as a +load balancer, to route traffic from the internet to your application. +Install the NGINX Ingress Controller +through the GitLab [Cluster management project template](../../../user/clusters/management_project_template.md), +or manually with Google Cloud Shell: + +1. Go to your cluster's details page, and select the **Advanced Settings** tab. +1. Select the link to Google Kubernetes Engine to visit the cluster on Google Cloud Console. +1. On the GKE cluster page, select **Connect**, then select **Run in Cloud Shell**. +1. After the Cloud Shell starts, run these commands to install NGINX Ingress Controller: + + ```shell + kubectl create ns gitlab-managed-apps + helm repo add stable https://charts.helm.sh/stable + helm repo update + helm install ingress stable/nginx-ingress -n gitlab-managed-apps + + # Check that the ingress controller is installed successfully + kubectl get service ingress-nginx-ingress-controller -n gitlab-managed-apps + ``` + +## Configure your base domain + +Follow these steps to configure the base domain where you access your apps. + +1. A few minutes after you install NGINX, the load balancer obtains an IP address, and you can + get the external IP address with the following command: + + ```shell + kubectl get service ingress-nginx-ingress-controller -n gitlab-managed-apps -ojson | jq -r '.status.loadBalancer.ingress[].ip' + ``` + + Replace `gitlab-managed-apps` if you have overwritten your namespace. + + Copy this IP address, as you need it in the next step. + +1. Go back to the cluster page on GitLab, and go to the **Details** tab. + - Add your **Base domain**. For this example, use the domain `<IP address>.nip.io`. + - Select **Save changes**. + +  + +## Enable Auto DevOps (optional) + +While Auto DevOps is enabled by default, Auto DevOps can be disabled at both +the instance level (for self-managed instances) and the group level. Complete +these steps to enable Auto DevOps if it's disabled: + +1. Go to **Settings > CI/CD > Auto DevOps**, and select **Expand**. +1. Select **Default to Auto DevOps pipeline** to display more options. +1. In **Deployment strategy**, select your desired [continuous deployment strategy](../requirements.md#auto-devops-deployment-strategy) + to deploy the application to production after the pipeline successfully runs on the default branch. +1. Select **Save changes**. + +  + +After you save your changes, GitLab creates a new pipeline. To view it, go to +**{rocket}** **CI/CD > Pipelines**. + +In the next section, we explain what each job does in the pipeline. + +## Deploy the application + +When your pipeline runs, what is it doing? + +To view the jobs in the pipeline, select the pipeline's status badge. The +**{status_running}** icon displays when pipeline jobs are running, and updates +without refreshing the page to **{status_success}** (for success) or +**{status_failed}** (for failure) when the jobs complete. + +The jobs are separated into stages: + + + +- **Build** - The application builds a Docker image and uploads it to your project's + [Container Registry](../../../user/packages/container_registry/index.md) ([Auto Build](../stages.md#auto-build)). +- **Test** - GitLab runs various checks on the application, but all jobs except `test` + are allowed to fail in the test stage: + + - The `test` job runs unit and integration tests by detecting the language and + framework ([Auto Test](../stages.md#auto-test)) + - The `code_quality` job checks the code quality and is allowed to fail + ([Auto Code Quality](../stages.md#auto-code-quality)) + - The `container_scanning` job checks the Docker container if it has any + vulnerabilities and is allowed to fail ([Auto Container Scanning](../stages.md#auto-container-scanning)) + - The `dependency_scanning` job checks if the application has any dependencies + susceptible to vulnerabilities and is allowed to fail + ([Auto Dependency Scanning](../stages.md#auto-dependency-scanning)) + - Jobs suffixed with `-sast` run static analysis on the current code to check for potential + security issues, and are allowed to fail ([Auto SAST](../stages.md#auto-sast)) + - The `secret-detection` job checks for leaked secrets and is allowed to fail ([Auto Secret Detection](../stages.md#auto-secret-detection)) + - The `license_scanning` job searches the application's dependencies to determine each of their + licenses and is allowed to fail + ([Auto License Compliance](../stages.md#auto-license-compliance)) + +- **Review** - Pipelines on the default branch include this stage with a `dast_environment_deploy` job. + To learn more, see [Dynamic Application Security Testing (DAST)](../../../user/application_security/dast/index.md). + +- **Production** - After the tests and checks finish, the application deploys in + Kubernetes ([Auto Deploy](../stages.md#auto-deploy)). + +- **Performance** - Performance tests are run on the deployed application + ([Auto Browser Performance Testing](../stages.md#auto-browser-performance-testing)). + +- **Cleanup** - Pipelines on the default branch include this stage with a `stop_dast_environment` job. + +After running a pipeline, you should view your deployed website and learn how +to monitor it. + +### Monitor your project + +After successfully deploying your application, you can view its website and check +on its health on the **Environments** page by navigating to +**Deployments > Environments**. This page displays details about +the deployed applications, and the right-hand column displays icons that link +you to common environment tasks: + + + +- **Open live environment** (**{external-link}**) - Opens the URL of the application deployed in production +- **Monitoring** (**{chart}**) - Opens the metrics page where Prometheus collects data + about the Kubernetes cluster and how the application + affects it in terms of memory usage, CPU usage, and latency +- **Deploy to** (**{play}** **{chevron-lg-down}**) - Displays a list of environments you can deploy to +- **Terminal** (**{terminal}**) - Opens a [web terminal](../../../ci/environments/index.md#web-terminals-deprecated) + session inside the container where the application is running +- **Re-deploy to environment** (**{repeat}**) - For more information, see + [Retrying and rolling back](../../../ci/environments/index.md#retry-or-roll-back-a-deployment) +- **Stop environment** (**{stop}**) - For more information, see + [Stopping an environment](../../../ci/environments/index.md#stop-an-environment) + +GitLab displays the [deploy board](../../../user/project/deploy_boards.md) below the +environment's information, with squares representing pods in your +Kubernetes cluster, color-coded to show their status. Hovering over a square on +the deploy board displays the state of the deployment, and selecting the square +takes you to the pod's logs page. + +NOTE: +The example shows only one pod hosting the application at the moment, but you can add +more pods by defining the [`REPLICAS` CI/CD variable](../customize.md#cicd-variables) +in **Settings > CI/CD > Variables**. + +### Work with branches + +Following the [GitLab flow](../../gitlab_flow.md#working-with-feature-branches), +you should next create a feature branch to add content to your application: + +1. In your project's repository, go to the following file: `app/views/welcome/index.html.erb`. + This file should only contain a paragraph: `<p>You're on Rails!</p>`. +1. Open the GitLab [Web IDE](../../../user/project/web_ide/index.md) to make the change. +1. Edit the file so it contains: + + ```html + <p>You're on Rails! Powered by GitLab Auto DevOps.</p> + ``` + +1. Stage the file. Add a commit message, then create a new branch and a merge request + by selecting **Commit**. + +  + +After submitting the merge request, GitLab runs your pipeline, and all the jobs +in it, as [described previously](#deploy-the-application), in addition to +a few more that run only on branches other than the default branch. + + + +After a few minutes a test fails, which means a test was +'broken' by your change. Select the failed `test` job to see more information +about it: + +```plaintext +Failure: +WelcomeControllerTest#test_should_get_index [/app/test/controllers/welcome_controller_test.rb:7]: +<You're on Rails!> expected but was +<You're on Rails! Powered by GitLab Auto DevOps.>.. +Expected 0 to be >= 1. + +bin/rails test test/controllers/welcome_controller_test.rb:4 +``` + +To fix the broken test: + +1. Return to your merge request. +1. In the upper right corner, select **Code**, then select **Open in Gitpod**. +1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` + file, and select it to open it. +1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` +1. Select **Commit**. +1. In the left-hand column, under **Unstaged changes**, select the checkmark icon + (**{stage-all}**) to stage the changes. +1. Write a commit message, and select **Commit**. + +Return to the **Overview** page of your merge request, and you should not only +see the test passing, but also the application deployed as a +[review application](../stages.md#auto-review-apps). You can visit it by selecting +the **View app** **{external-link}** button to see your changes deployed. + + + +After merging the merge request, GitLab runs the pipeline on the default branch, +and then deploys the application to production. + +## Conclusion + +After implementing this project, you should have a solid understanding of the basics of Auto DevOps. +You started from building and testing, to deploying and monitoring an application +all in GitLab. Despite its automatic nature, Auto DevOps can also be configured +and customized to fit your workflow. Here are some helpful resources for further reading: + +1. [Auto DevOps](../index.md) +1. [Multiple Kubernetes clusters](../multiple_clusters_auto_devops.md) +1. [Incremental rollout to production](../customize.md#incremental-rollout-to-production) +1. [Disable jobs you don't need with CI/CD variables](../customize.md#cicd-variables) +1. [Use your own buildpacks to build your application](../customize.md#custom-buildpacks) +1. [Prometheus monitoring](../../../user/project/integrations/prometheus.md) diff --git a/doc/topics/autodevops/img/guide_base_domain_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_base_domain_v12_3.png Binary files differindex 7d3b6a2f905..7d3b6a2f905 100644 --- a/doc/topics/autodevops/img/guide_base_domain_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_base_domain_v12_3.png diff --git a/doc/topics/autodevops/img/guide_create_project_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_create_project_v12_3.png Binary files differindex a22730520ef..a22730520ef 100644 --- a/doc/topics/autodevops/img/guide_create_project_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_create_project_v12_3.png diff --git a/doc/topics/autodevops/img/guide_enable_autodevops_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_enable_autodevops_v12_3.png Binary files differindex a3bcaeb99ae..a3bcaeb99ae 100644 --- a/doc/topics/autodevops/img/guide_enable_autodevops_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_enable_autodevops_v12_3.png diff --git a/doc/topics/autodevops/img/guide_environments_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_environments_v12_3.png Binary files differindex 87253f9929d..87253f9929d 100644 --- a/doc/topics/autodevops/img/guide_environments_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_environments_v12_3.png diff --git a/doc/topics/autodevops/img/guide_ide_commit_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_ide_commit_v12_3.png Binary files differindex afea0dc8fb4..afea0dc8fb4 100644 --- a/doc/topics/autodevops/img/guide_ide_commit_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_ide_commit_v12_3.png diff --git a/doc/topics/autodevops/img/guide_merge_request_review_app_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_merge_request_review_app_v12_3.png Binary files differindex e94654f4e50..e94654f4e50 100644 --- a/doc/topics/autodevops/img/guide_merge_request_review_app_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_merge_request_review_app_v12_3.png diff --git a/doc/topics/autodevops/img/guide_merge_request_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_merge_request_v12_3.png Binary files differindex 5565be701cd..5565be701cd 100644 --- a/doc/topics/autodevops/img/guide_merge_request_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_merge_request_v12_3.png diff --git a/doc/topics/autodevops/img/guide_pipeline_stages_v13_0.png b/doc/topics/autodevops/cloud_deployments/img/guide_pipeline_stages_v13_0.png Binary files differindex fb102879556..fb102879556 100644 --- a/doc/topics/autodevops/img/guide_pipeline_stages_v13_0.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_pipeline_stages_v13_0.png diff --git a/doc/topics/autodevops/img/guide_project_landing_page_v12_10.png b/doc/topics/autodevops/cloud_deployments/img/guide_project_landing_page_v12_10.png Binary files differindex 54e7141dad2..54e7141dad2 100644 --- a/doc/topics/autodevops/img/guide_project_landing_page_v12_10.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_project_landing_page_v12_10.png diff --git a/doc/topics/autodevops/img/guide_project_template_v12_3.png b/doc/topics/autodevops/cloud_deployments/img/guide_project_template_v12_3.png Binary files differindex 2b8d7224747..2b8d7224747 100644 --- a/doc/topics/autodevops/img/guide_project_template_v12_3.png +++ b/doc/topics/autodevops/cloud_deployments/img/guide_project_template_v12_3.png diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index e2d984dbbff..dfc2828e383 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -409,6 +409,7 @@ applications. | `AUTO_DEVOPS_BUILD_IMAGE_CNB_BUILDER` | The builder used when building with Cloud Native Buildpacks. The default builder is `heroku/buildpacks:18`. [More details](stages.md#auto-build-using-cloud-native-buildpacks). | | `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes doesn't prevent word splitting. [More details](#passing-arguments-to-docker-build). | | `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI/CD variable names](#forward-cicd-variables-to-the-build-environment) to be forwarded to the build environment (the buildpack builder or `docker build`). | +| `AUTO_DEVOPS_BUILD_IMAGE_CNB_PORT` | In GitLab 15.0 and later, port exposed by the generated Docker image. Set to `false` to prevent exposing any ports. Defaults to `5000`. | | `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app). | | `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. | | `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | Used to set the name of the Helm repository. Defaults to `gitlab`. | @@ -466,6 +467,8 @@ The following table lists CI/CD variables related to the database. | `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.16` for tests and deployments as of GitLab 13.0 (previously `9.6.2`). If `AUTO_DEVOPS_POSTGRES_CHANNEL` is set to `1`, deployments uses the default version `9.6.2`. | | `POSTGRES_HELM_UPGRADE_VALUES_FILE` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows the `helm upgrade` values file for PostgreSQL to be overridden. Defaults to `.gitlab/auto-deploy-postgres-values.yaml`. | | `POSTGRES_HELM_UPGRADE_EXTRA_ARGS` | In GitLab 13.8 and later, and when using [auto-deploy-image v2](upgrading_auto_deploy_dependencies.md), this variable allows extra PostgreSQL options in `helm upgrade` commands when deploying the application. Note that using quotes doesn't prevent word splitting. | +| `POSTGRES_CHART_REPOSITORY` | Helm Chart repository used to search for PostgreSQL chart. Defaults to `https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami`. | +| `POSTGRES_CHART_VERSION` | Helm Chart version used for PostgreSQL chart. Defaults to `8.2.1`. | ### Disable jobs @@ -669,7 +672,7 @@ The percentage is based on the `REPLICAS` CI/CD variable, and defines the number pods you want to have for your deployment. If the value is `10`, and you run the `10%` rollout job, there is `1` new pod and `9` old ones. -To start a job, click the play icon (**{play}**) next to the job's name. You're not +To start a job, select the play icon (**{play}**) next to the job's name. You're not required to go from `10%` to `100%`, you can jump to whatever job you want. You can also scale down by running a lower percentage job, just before hitting `100%`. Once you get to `100%`, you can't scale down, and you'd have to roll diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index c8efc40a4cd..6d4f6eb698e 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -234,16 +234,9 @@ and clear the **Default to Auto DevOps pipeline** checkbox. ### Quick start -To guide you through the process of setting up Auto DevOps to deploy to a Kubernetes cluster on -Google Kubernetes Engine (GKE), see the [quick start guide](quick_start_guide.md). - -You can also follow the quick start for the general steps, but deploy to -[AWS ECS](requirements.md#auto-devops-requirements-for-amazon-ecs) instead. - -If you're a self-managed user, before deploying to GKE, a GitLab administrator needs to: - -1. Configure the [Google OAuth 2.0 OmniAuth Provider](../../integration/google.md). -1. Configure a cluster on GKE. +- [Use Auto DevOps to deploy to a Kubernetes cluster on Google Kubernetes Engine (GKE)](cloud_deployments/auto_devops_with_gke.md) +- [Use Auto DevOps to deploy to EC2](cloud_deployments/auto_devops_with_ec2.md) +- [Use Auto DevOps to deploy to ECS](cloud_deployments/auto_devops_with_ecs.md) ## Upgrade Auto DevOps dependencies when updating GitLab diff --git a/doc/topics/autodevops/multiple_clusters_auto_devops.md b/doc/topics/autodevops/multiple_clusters_auto_devops.md index 8156ae7c7ac..9c766385e66 100644 --- a/doc/topics/autodevops/multiple_clusters_auto_devops.md +++ b/doc/topics/autodevops/multiple_clusters_auto_devops.md @@ -35,7 +35,8 @@ To add a different cluster for each environment: 1. Navigate to your project's **Infrastructure > Kubernetes clusters**. 1. Create the Kubernetes clusters with their respective environment scope, as described from the table above. -1. After creating the clusters, navigate to each cluster and [install Ingress](quick_start_guide.md#install-ingress). +1. After creating the clusters, navigate to each cluster and + [install Ingress](cloud_deployments/auto_devops_with_gke.md#install-ingress). Wait for the Ingress IP address to be assigned. 1. Make sure you've [configured your DNS](requirements.md#auto-devops-base-domain) with the specified Auto DevOps domains. diff --git a/doc/topics/autodevops/prepare_deployment.md b/doc/topics/autodevops/prepare_deployment.md index 7c9bf5a770e..22a50df7f54 100644 --- a/doc/topics/autodevops/prepare_deployment.md +++ b/doc/topics/autodevops/prepare_deployment.md @@ -51,7 +51,7 @@ as other environment [variables](../../ci/variables/index.md#cicd-variable-prece If you don't specify the base domain in your projects and groups, Auto DevOps uses the instance-wide **Auto DevOps domain**. -Auto DevOps requires a wildcard DNS A record matching the base domains. For +Auto DevOps requires a wildcard DNS `A` record matching the base domains. For a base domain of `example.com`, you'd need a DNS entry like: ```plaintext diff --git a/doc/topics/autodevops/quick_start_guide.md b/doc/topics/autodevops/quick_start_guide.md index 8d1bf7adc7f..a524b6e6451 100644 --- a/doc/topics/autodevops/quick_start_guide.md +++ b/doc/topics/autodevops/quick_start_guide.md @@ -1,331 +1,11 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../autodevops/cloud_deployments/auto_devops_with_gke.md' +remove_date: '2022-09-18' --- -# Tutorial: Use Auto DevOps to deploy an application to Google Kubernetes Engine **(FREE)** +This document was moved to [another location](../autodevops/cloud_deployments/auto_devops_with_gke.md). -In this tutorial, we'll help you to get started with [Auto DevOps](index.md) -through an example of how to deploy an application to Google Kubernetes Engine (GKE). - -You are using the GitLab native Kubernetes integration, so you don't need -to create a Kubernetes cluster manually using the Google Cloud Platform console. -You are creating and deploying an application that you create from a GitLab template. - -These instructions also work for self-managed GitLab instances. -Ensure your own [runners are configured](../../ci/runners/index.md) and -[Google OAuth is enabled](../../integration/google.md). - -To deploy a project to Google Kubernetes Engine, follow the steps below: - -1. [Configure your Google account](#configure-your-google-account) -1. [Create a new project from a template](#create-a-new-project-from-a-template) -1. [Create a Kubernetes cluster from GitLab](#create-a-kubernetes-cluster-from-gitlab) -1. [Install Ingress](#install-ingress) -1. [Configure your base domain](#configure-your-base-domain) -1. [Enable Auto DevOps](#enable-auto-devops-optional) -1. [Deploy the application](#deploy-the-application) - -## Configure your Google account - -Before creating and connecting your Kubernetes cluster to your GitLab project, -you need a [Google Cloud Platform account](https://console.cloud.google.com). -Sign in with an existing Google account, such as the one you use to access Gmail -or Google Drive, or create a new one. - -1. Follow the steps described in the ["Before you begin" section](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) - of the Kubernetes Engine documentation to enable the required APIs and related services. -1. Ensure you've created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) - with Google Cloud Platform. - -NOTE: -Every new Google Cloud Platform (GCP) account receives [$300 in credit](https://console.cloud.google.com/freetrial), -and in partnership with Google, GitLab is able to offer an additional $200 for new -GCP accounts to get started with the GitLab integration with Google Kubernetes Engine. -[Follow this link](https://cloud.google.com/partners/partnercredit/?pcn_code=0014M00001h35gDQAQ#contact-form) -and apply for credit. - -## Create a new project from a template - -Use a GitLab project template to get started. As the name suggests, -those projects provide a bare-bones application built on some well-known frameworks. - -1. On the top bar in GitLab, select the plus icon (**{plus-square}**), and select - **New project/repository**. -1. Go to the **Create from template** tab, where you can choose a Ruby on - Rails, Spring, or NodeJS Express project. - For this tutorial, use the Ruby on Rails template. - -  - -1. Give your project a name, optionally a description, and make it public so that - you can take advantage of the features available in the - [GitLab Ultimate plan](https://about.gitlab.com/pricing/). - -  - -1. Select **Create project**. - -Now that you've created a project, create the Kubernetes cluster -to deploy this project to. - -## Create a Kubernetes cluster from GitLab - -1. On your project's landing page, select the button **Add Kubernetes cluster**. - -  - -1. On the **Kubernetes clusters** page, select the **Create a new cluster** option from the **Actions** dropdown menu. - -1. On the **Connect a Kubernetes cluster** page, select **Google GKE**. - -1. Connect with your Google account, and select **Allow** to allow access to your - Google account. (This authorization request is only displayed the first time - you connect GitLab with your Google account.) - - After authorizing access, the **Connect a Kubernetes cluster** page - is displayed. - -1. In the **Enter your Kubernetes cluster certificate details** section, provide - details about your cluster: - - - **Kubernetes cluster name** - - **Environment scope** - Leave this field unchanged. - - **Google Cloud Platform project** - Select a project. When you - [configured your Google account](#configure-your-google-account), a project - should have already been created for you. - - **Zone** - The [region/zone](https://cloud.google.com/compute/docs/regions-zones/) to - create the cluster in. - - **Number of nodes** - - **Machine type** - For more information about - [machine types](https://cloud.google.com/compute/docs/machine-types), see Google's documentation. - - **Enable Cloud Run for Anthos** - Select this checkbox to use the - [Cloud Run](../../user/project/clusters/add_gke_clusters.md#cloud-run-for-anthos), - Istio, and HTTP Load Balancing add-ons for this cluster. - - **GitLab-managed cluster** - Select this checkbox to - [allow GitLab to manage namespace and service accounts](../../user/project/clusters/gitlab_managed_clusters.md) for this cluster. - -1. Select **Create Kubernetes cluster**. - -After a couple of minutes, the cluster is created. You can also see its -status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). - -## Install Ingress - -After your cluster is running, you must install NGINX Ingress Controller as a -load balancer, to route traffic from the internet to your application. -Install the NGINX Ingress Controller -through the GitLab [Cluster management project template](../../user/clusters/management_project_template.md), -or manually with Google Cloud Shell: - -1. Go to your cluster's details page, and select the **Advanced Settings** tab. -1. Select the link to Google Kubernetes Engine to visit the cluster on Google Cloud Console. -1. On the GKE cluster page, select **Connect**, then select **Run in Cloud Shell**. -1. After the Cloud Shell starts, run these commands to install NGINX Ingress Controller: - - ```shell - kubectl create ns gitlab-managed-apps - helm repo add stable https://charts.helm.sh/stable - helm repo update - helm install ingress stable/nginx-ingress -n gitlab-managed-apps - - # Check that the ingress controller is installed successfully - kubectl get service ingress-nginx-ingress-controller -n gitlab-managed-apps - ``` - -## Configure your base domain - -Follow these steps to configure the base domain where you access your apps. - -1. A few minutes after you install NGINX, the load balancer obtains an IP address, and you can - get the external IP address with the following command: - - ```shell - kubectl get service ingress-nginx-ingress-controller -n gitlab-managed-apps -ojson | jq -r '.status.loadBalancer.ingress[].ip' - ``` - - Replace `gitlab-managed-apps` if you have overwritten your namespace. - - Copy this IP address, as you need it in the next step. - -1. Go back to the cluster page on GitLab, and go to the **Details** tab. - - Add your **Base domain**. For this example, use the domain `<IP address>.nip.io`. - - Select **Save changes**. - -  - -## Enable Auto DevOps (optional) - -While Auto DevOps is enabled by default, Auto DevOps can be disabled at both -the instance level (for self-managed instances) and the group level. Complete -these steps to enable Auto DevOps if it's disabled: - -1. Go to **Settings > CI/CD > Auto DevOps**, and select **Expand**. -1. Select **Default to Auto DevOps pipeline** to display more options. -1. In **Deployment strategy**, select your desired [continuous deployment strategy](requirements.md#auto-devops-deployment-strategy) - to deploy the application to production after the pipeline successfully runs on the default branch. -1. Select **Save changes**. - -  - -After you save your changes, GitLab creates a new pipeline. To view it, go to -**{rocket}** **CI/CD > Pipelines**. - -In the next section, we explain what each job does in the pipeline. - -## Deploy the application - -When your pipeline runs, what is it doing? - -To view the jobs in the pipeline, select the pipeline's status badge. The -**{status_running}** icon displays when pipeline jobs are running, and updates -without refreshing the page to **{status_success}** (for success) or -**{status_failed}** (for failure) when the jobs complete. - -The jobs are separated into stages: - - - -- **Build** - The application builds a Docker image and uploads it to your project's - [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](stages.md#auto-build)). -- **Test** - GitLab runs various checks on the application, but all jobs except `test` - are allowed to fail in the test stage: - - - The `test` job runs unit and integration tests by detecting the language and - framework ([Auto Test](stages.md#auto-test)) - - The `code_quality` job checks the code quality and is allowed to fail - ([Auto Code Quality](stages.md#auto-code-quality)) - - The `container_scanning` job checks the Docker container if it has any - vulnerabilities and is allowed to fail ([Auto Container Scanning](stages.md#auto-container-scanning)) - - The `dependency_scanning` job checks if the application has any dependencies - susceptible to vulnerabilities and is allowed to fail - ([Auto Dependency Scanning](stages.md#auto-dependency-scanning)) - - Jobs suffixed with `-sast` run static analysis on the current code to check for potential - security issues, and are allowed to fail ([Auto SAST](stages.md#auto-sast)) - - The `secret-detection` job checks for leaked secrets and is allowed to fail ([Auto Secret Detection](stages.md#auto-secret-detection)) - - The `license_scanning` job searches the application's dependencies to determine each of their - licenses and is allowed to fail - ([Auto License Compliance](stages.md#auto-license-compliance)) - -- **Review** - Pipelines on the default branch include this stage with a `dast_environment_deploy` job. - To learn more, see [Dynamic Application Security Testing (DAST)](../../user/application_security/dast/index.md). - -- **Production** - After the tests and checks finish, the application deploys in - Kubernetes ([Auto Deploy](stages.md#auto-deploy)). - -- **Performance** - Performance tests are run on the deployed application - ([Auto Browser Performance Testing](stages.md#auto-browser-performance-testing)). - -- **Cleanup** - Pipelines on the default branch include this stage with a `stop_dast_environment` job. - -After running a pipeline, you should view your deployed website and learn how -to monitor it. - -### Monitor your project - -After successfully deploying your application, you can view its website and check -on its health on the **Environments** page by navigating to -**Deployments > Environments**. This page displays details about -the deployed applications, and the right-hand column displays icons that link -you to common environment tasks: - - - -- **Open live environment** (**{external-link}**) - Opens the URL of the application deployed in production -- **Monitoring** (**{chart}**) - Opens the metrics page where Prometheus collects data - about the Kubernetes cluster and how the application - affects it in terms of memory usage, CPU usage, and latency -- **Deploy to** (**{play}** **{angle-down}**) - Displays a list of environments you can deploy to -- **Terminal** (**{terminal}**) - Opens a [web terminal](../../ci/environments/index.md#web-terminals-deprecated) - session inside the container where the application is running -- **Re-deploy to environment** (**{repeat}**) - For more information, see - [Retrying and rolling back](../../ci/environments/index.md#retry-or-roll-back-a-deployment) -- **Stop environment** (**{stop}**) - For more information, see - [Stopping an environment](../../ci/environments/index.md#stop-an-environment) - -GitLab displays the [deploy board](../../user/project/deploy_boards.md) below the -environment's information, with squares representing pods in your -Kubernetes cluster, color-coded to show their status. Hovering over a square on -the deploy board displays the state of the deployment, and selecting the square -takes you to the pod's logs page. - -NOTE: -The example shows only one pod hosting the application at the moment, but you can add -more pods by defining the [`REPLICAS` CI/CD variable](customize.md#cicd-variables) -in **Settings > CI/CD > Variables**. - -### Work with branches - -Following the [GitLab flow](../gitlab_flow.md#working-with-feature-branches), -you should next create a feature branch to add content to your application: - -1. In your project's repository, go to the following file: `app/views/welcome/index.html.erb`. - This file should only contain a paragraph: `<p>You're on Rails!</p>`. -1. Open the GitLab [Web IDE](../../user/project/web_ide/index.md) to make the change. -1. Edit the file so it contains: - - ```html - <p>You're on Rails! Powered by GitLab Auto DevOps.</p> - ``` - -1. Stage the file. Add a commit message, then create a new branch and a merge request - by selecting **Commit**. - -  - -After submitting the merge request, GitLab runs your pipeline, and all the jobs -in it, as [described previously](#deploy-the-application), in addition to -a few more that run only on branches other than the default branch. - - - -After a few minutes a test fails, which means a test was -'broken' by your change. Select the failed `test` job to see more information -about it: - -```plaintext -Failure: -WelcomeControllerTest#test_should_get_index [/app/test/controllers/welcome_controller_test.rb:7]: -<You're on Rails!> expected but was -<You're on Rails! Powered by GitLab Auto DevOps.>.. -Expected 0 to be >= 1. - -bin/rails test test/controllers/welcome_controller_test.rb:4 -``` - -To fix the broken test: - -1. Return to your merge request. -1. In the upper right corner, select **Code**, then select **Open in Gitpod**. -1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` - file, and select it to open it. -1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` -1. Select **Commit**. -1. In the left-hand column, under **Unstaged changes**, select the checkmark icon - (**{stage-all}**) to stage the changes. -1. Write a commit message, and select **Commit**. - -Return to the **Overview** page of your merge request, and you should not only -see the test passing, but also the application deployed as a -[review application](stages.md#auto-review-apps). You can visit it by selecting -the **View app** **{external-link}** button to see your changes deployed. - - - -After merging the merge request, GitLab runs the pipeline on the default branch, -and then deploys the application to production. - -## Conclusion - -After implementing this project, you should have a solid understanding of the basics of Auto DevOps. -You started from building and testing, to deploying and monitoring an application -all in GitLab. Despite its automatic nature, Auto DevOps can also be configured -and customized to fit your workflow. Here are some helpful resources for further reading: - -1. [Auto DevOps](index.md) -1. [Multiple Kubernetes clusters](multiple_clusters_auto_devops.md) -1. [Incremental rollout to production](customize.md#incremental-rollout-to-production) -1. [Disable jobs you don't need with CI/CD variables](customize.md#cicd-variables) -1. [Use your own buildpacks to build your application](customize.md#custom-buildpacks) -1. [Prometheus monitoring](../../user/project/integrations/prometheus.md) +<!-- This redirect file can be deleted after <2022-09-18>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md index 039d98efd47..02485ebafff 100644 --- a/doc/topics/autodevops/requirements.md +++ b/doc/topics/autodevops/requirements.md @@ -14,17 +14,16 @@ To prepare the deployment: 1. Define the [deployment strategy](#auto-devops-deployment-strategy). 1. Prepare the [base domain](#auto-devops-base-domain). -1. Define where you want to deploy it to: +1. Define where you want to deploy it: 1. [Kubernetes](#auto-devops-requirements-for-kubernetes). - 1. [Amazon Elastic Container Service (ECS)](#auto-devops-requirements-for-amazon-ecs). - 1. [Amazon EC2](#auto-devops-requirements-for-amazon-ec2). + 1. [Amazon Elastic Container Service (ECS)](cloud_deployments/auto_devops_with_ecs.md). + 1. [Amazon Elastic Kubernetes Service (EKS)](https://about.gitlab.com/blog/2020/05/05/deploying-application-eks/). + 1. [Amazon EC2](cloud_deployments/auto_devops_with_ec2.md). + 1. [Google Kubernetes Engine](cloud_deployments/auto_devops_with_gke.md). 1. [Bare metal](#auto-devops-requirements-for-bare-metal). -When done: - 1. [Enable Auto DevOps](index.md#enable-or-disable-auto-devops). -1. See the [quick start](quick_start_guide.md) process. ## Auto DevOps deployment strategy @@ -173,50 +172,6 @@ are skipped. After all requirements are met, you can [enable Auto DevOps](index.md#enable-or-disable-auto-devops). -## Auto DevOps requirements for Amazon ECS - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208132) in GitLab 13.0. - -You can choose to target [AWS ECS](../../ci/cloud_deployment/index.md) as a deployment platform instead of using Kubernetes. - -To get started on Auto DevOps to AWS ECS, you must add a specific CI/CD variable. -To do so, follow these steps: - -1. In GitLab, on the top bar, select **Menu > Projects** and find your project. -1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **Auto DevOps**. -1. Specify which AWS platform to target during the Auto DevOps deployment - by adding the `AUTO_DEVOPS_PLATFORM_TARGET` variable with one of the following values: - - `FARGATE` if the service you're targeting must be of launch type FARGATE. - - `ECS` if you're not enforcing any launch type check when deploying to ECS. - -When you trigger a pipeline, if you have Auto DevOps enabled and if you have correctly -[entered AWS credentials as variables](../../ci/cloud_deployment/index.md#deploy-your-application-to-the-aws-elastic-container-service-ecs), -your application is deployed to AWS ECS. - -If you have both a valid `AUTO_DEVOPS_PLATFORM_TARGET` variable and a Kubernetes cluster tied to your project, -only the deployment to Kubernetes runs. - -WARNING: -Setting the `AUTO_DEVOPS_PLATFORM_TARGET` variable to `ECS` triggers jobs -defined in the [`Jobs/Deploy/ECS.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml). -However, it's not recommended to [include](../../ci/yaml/index.md#includetemplate) -it on its own. This template is designed to be used with Auto DevOps only. It may change -unexpectedly causing your pipeline to fail if included on its own. Also, the job -names within this template may also change. Do not override these jobs' names in your -own pipeline, as the override stops working when the name changes. - -## Auto DevOps requirements for Amazon EC2 - -[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216008) in GitLab 13.6. - -You can target [AWS EC2](../../ci/cloud_deployment/index.md) -as a deployment platform instead of Kubernetes. To use Auto DevOps with AWS EC2, you must add a -specific CI/CD variable. - -For more details, see [Custom build job for Auto DevOps](../../ci/cloud_deployment/index.md#custom-build-job-for-auto-devops) -for deployments to AWS EC2. - ## Auto DevOps requirements for bare metal According to the [Kubernetes Ingress-NGINX docs](https://kubernetes.github.io/ingress-nginx/deploy/baremetal/): diff --git a/doc/topics/git/feature_branch_development.md b/doc/topics/git/feature_branch_development.md index ae1485741a5..b205cedbdaa 100644 --- a/doc/topics/git/feature_branch_development.md +++ b/doc/topics/git/feature_branch_development.md @@ -75,7 +75,7 @@ In this case, the feature branch would be `release-X-Y`. Assuming the `release-X  -1. After you click **Create merge request**, an option to **Change branches** displays. Select that option. +1. After you select **Create merge request**, an option to **Change branches** displays. Select that option. 1. In the **New Merge Request** screen, you can now select the **Source** and **Target** branches. In the screenshot shown, @@ -83,7 +83,7 @@ we have selected `test-branch` as the source, and `release-13-0` as the target.  -1. Once you've selected the Source and Target branches, click **Compare branches and continue**. +1. Once you've selected the Source and Target branches, select **Compare branches and continue**. You should see an entry similar to: ```plaintext @@ -94,7 +94,7 @@ we have selected `test-branch` as the source, and `release-13-0` as the target. An entry like this confirms your merge request's destination. -1. Make any additional changes in the **New Merge Request** screen, and click **Submit merge request**. +1. Make any additional changes in the **New Merge Request** screen, and select **Submit merge request**. 1. In the new merge request, look for **Request to merge**. An entry similar to this displays: ```plaintext diff --git a/doc/topics/git/feature_branching.md b/doc/topics/git/feature_branching.md index d3b2510f4e8..4e9b43d66f8 100644 --- a/doc/topics/git/feature_branching.md +++ b/doc/topics/git/feature_branching.md @@ -16,7 +16,7 @@ comments: false ## Feature branching sample workflow -1. Create a new feature branch called 'squash_some_bugs' +1. Create a new feature branch called `squash_some_bugs` 1. Edit '`bugs.rb`' and remove all the bugs. 1. Commit 1. Push diff --git a/doc/topics/git/git_rebase.md b/doc/topics/git/git_rebase.md index 87fce0b29ff..1e118f98ca5 100644 --- a/doc/topics/git/git_rebase.md +++ b/doc/topics/git/git_rebase.md @@ -180,7 +180,7 @@ the operation you want to perform in each commit. To do so, edit the commits in your terminal's text editor. For example, with [Vim](https://www.vim.org/) as the text editor in -a macOS's `ZSH` shell, you can `squash` or `fixup` (combine) all three commits: +a macOS's Zsh shell, you can `squash` or `fixup` (combine) all three commits: <!-- vale gitlab.FirstPerson = NO --> diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index 410d2150de5..d40690d86c6 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -29,7 +29,7 @@ Documentation for GitLab instance administrators is under [LFS administration do ## Requirements -- Git LFS must be [enabled in project settings](../../../user/project/settings/index.md#sharing-and-permissions). +- Git LFS must be [enabled in project settings](../../../user/project/settings/index.md#configure-project-visibility-features-and-permissions). - [Git LFS client](https://git-lfs.github.com) version 1.0.1 or higher must be installed. ## Known limitations @@ -198,11 +198,7 @@ If the status `error 501` is shown, it is because: remove the line and try to update your Git LFS client. Only version 1.0.1 and newer are supported. -<!-- vale gitlab.Spelling = NO --> - -### getsockopt: connection refused - -<!-- vale gitlab.Spelling = YES --> +### `getsockopt: connection refused` If you push an LFS object to a project and receive an error like this, the LFS client is trying to reach GitLab through HTTPS. However, your GitLab @@ -262,11 +258,7 @@ If you are storing LFS files outside of GitLab you can disable LFS on the projec It is possible to host LFS objects externally by setting a custom LFS URL with `git config -f .lfsconfig lfs.url https://example.com/<project>.git/info/lfs`. -<!-- vale gitlab.Spelling = NO --> - -You might choose to do this if you are using an appliance like a Sonatype Nexus to store LFS data. If you choose to use an external LFS store, +You might choose to do this if you are using an appliance like a Nexus Repository to store LFS data. If you choose to use an external LFS store, GitLab can't verify LFS objects. Pushes then fail if you have GitLab LFS support enabled. -<!-- vale gitlab.Spelling = YES --> - To stop push failure, LFS support can be disabled in the [Project settings](../../../user/project/settings/index.md), which also disables GitLab LFS value-adds (Verifying LFS objects, UI integration for LFS). diff --git a/doc/topics/git/lfs/migrate_to_git_lfs.md b/doc/topics/git/lfs/migrate_to_git_lfs.md index 32e3b6e2f72..864615e7264 100644 --- a/doc/topics/git/lfs/migrate_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_to_git_lfs.md @@ -121,7 +121,7 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- 1. Navigate to your project's **Settings > Repository** and expand **Protected branches**. - 1. Scroll down to locate the protected branches and click + 1. Scroll down to locate the protected branches and select **Unprotect** the default branch. 1. Force-push to GitLab: @@ -162,7 +162,7 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- 1. Select the default branch from the **Branch** dropdown menu, and set up the **Allowed to push** and **Allowed to merge** rules. - 1. Click **Protect**. + 1. Select **Protect**. <!-- ## Troubleshooting diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index 91ff4d69c2f..eeae0d78638 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -160,7 +160,7 @@ For more details, see the Git documentation for ``` WARNING: - Git integrations with `bash`, `zsh`, etc and editors that automatically + Git integrations with `bash`, Zsh, etc and editors that automatically show Git status information often run `git fetch` which fetches the entire repository. Disabling or reconfiguring these integrations might be required. diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index d35eba0d782..034c822671d 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -34,7 +34,7 @@ After getting used to these three steps, the next challenge is the branching mod Because many organizations new to Git have no conventions for how to work with it, their repositories can quickly become messy. The biggest problem is that many long-running branches emerge that all contain part of the changes. People have a hard time figuring out which branch has the latest code, or which branch to deploy to production. -Frequently, the reaction to this problem is to adopt a standardized pattern such as [Git flow](https://nvie.com/posts/a-successful-git-branching-model/) and [GitHub flow](http://scottchacon.com/2011/08/31/github-flow.html). +Frequently, the reaction to this problem is to adopt a standardized pattern such as [Git flow](https://nvie.com/posts/a-successful-git-branching-model/) and [GitHub flow](https://scottchacon.com/2011/08/31/github-flow.html). We think there is still room for improvement. In this document, we describe a set of practices we call GitLab flow. For a video introduction of how this works in GitLab, see [GitLab Flow](https://youtu.be/InKNIvky2KE). diff --git a/doc/topics/offline/index.md b/doc/topics/offline/index.md index a48ac9feb1a..13207c8c612 100644 --- a/doc/topics/offline/index.md +++ b/doc/topics/offline/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/topics/offline/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index 4426f955cb7..901a2dc750c 100644 --- a/doc/topics/offline/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -202,7 +202,7 @@ done. ### Disable Version Check and Service Ping The Version Check and Service Ping services improve the GitLab user experience and ensure that -users are on the most up-to-date instances of GitLab. These two services can be turned off for air-gapped +users are on the most up-to-date instances of GitLab. These two services can be turned off for offline environments so that they do not attempt and fail to reach out to GitLab services. Learn more about [disabling usage statistics](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics). diff --git a/doc/topics/set_up_organization.md b/doc/topics/set_up_organization.md index 877dbe95e70..c8183a0757e 100644 --- a/doc/topics/set_up_organization.md +++ b/doc/topics/set_up_organization.md @@ -10,7 +10,7 @@ Configure your organization and its users. Determine user roles and give everyone access to the projects they need. - [Members](../user/project/members/index.md) -- [Workspace](../user/workspace/index.md) _(Coming soon)_ +- [Workspace](../user/workspace/index.md) _(In development)_ - [Groups](../user/group/index.md) - [User account options](../user/profile/index.md) - [SSH keys](../user/ssh.md) diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index 55f8321b559..1b4762052c0 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -16,7 +16,7 @@ and running quickly. | Topic | Description | Good for beginners | |-------|-------------|--------------------| | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Introduction to GitLab](https://youtu.be/_4SmIyQ5eis?t=90) (59m 51s) | Walk through recommended processes and example workflows for using GitLab. | **{star}** | -| [GitLab 101](https://gitlab.edcast.com/pathways/copy-of-gitlab-certification) | Learn the basics of GitLab in this certification course. | **{star}** | +| [GitLab 101](https://gitlab.edcast.com/pathways/ECL-ce65e759-d9e7-459f-83d0-1765459395d2) | Learn the basics of GitLab in this certification course. | **{star}** | | <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Use GitLab for DevOps](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) (12m 34s) | Use GitLab through the entire DevOps lifecycle, from planning to monitoring. | **{star}** | | [Use Markdown at GitLab](../user/markdown.md) | GitLab Flavored Markdown (GLFM) is used in many areas of GitLab, for example, in merge requests. | **{star}** | | [GitLab 201](https://gitlab.edcast.com/pathways/ECL-44010cf6-7a9c-4b9b-b684-fa08508a3252) | Go beyond the basics to learn more about using GitLab for your work. | | diff --git a/doc/tutorials/move_personal_project_to_a_group.md b/doc/tutorials/move_personal_project_to_a_group.md new file mode 100644 index 00000000000..f8b91792780 --- /dev/null +++ b/doc/tutorials/move_personal_project_to_a_group.md @@ -0,0 +1,95 @@ +--- +stage: none +group: unassigned +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Move your personal project to a group **(FREE SAAS)** + +This tutorial will show you how to move a personal project to a group. + +## Why is a group important? + +In GitLab, you use [groups](../user/group/index.md) +to manage one or more related projects at the same time. +A group gives you some great benefits. For example, you can: + +- Manage permissions for your projects. +- View all of the issues and merge requests for the projects in the group. +- View all unique users in your namespace, across all projects. +- Manage usage quotas. +- Start a trial or upgrade to a paid tier. This option is important if you're + impacted by the [changes to user limits](https://about.gitlab.com/blog/2022/03/24/efficient-free-tier/), + and need more users. + +However, if you're working in a [personal project](../user/project/working_with_projects.md#view-personal-projects), +you can't use these features. Personal projects are created under your +[personal namespace](../user/group/index.md#namespaces). They're not part of a group, +so you can't get any of the benefits and features of a group. + +But don't worry! You can move your existing personal project to a group. +The next steps show you how. + +## Steps + +Here's an overview of what we're going to do: + +1. [Create a group](#create-a-group). +1. [Move your project to a group](#move-your-project-to-a-group). +1. [Work with your group](#work-with-your-group). + +### Create a group + +To begin, make sure you have a suitable group to move your project to. +The group must allow the creation of projects, and you must have at least the +Maintainer role for the group. + +If you don't have a group, create one: + +1. On the top bar, select **Menu > Groups > Create group** +1. Select **Create group**. +1. In **Group name**, enter a name for the group. +1. In **Group URL**, enter a path for the group, which is used as the namespace. +1. Choose the [visibility level](../user/public_access.md). +1. Optional. Fill in information to personalize your experience. +1. Select **Create group**. + +### Move your project to a group + +Before you move your project to a group: + +- You must have the Owner role for the project. +- Remove any [container images](../user/packages/container_registry/index.md#limitations) + and [NPM packages](../user/packages/npm_registry/index.md#limitations). + +Now you're ready to move your project: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. Under **Transfer project**, choose the group to transfer the project to. +1. Select **Transfer project**. +1. Enter the project's name and select **Confirm**. + +You are redirected to the project's new page. +If you have more than one personal project, you can repeat these steps for each +project. + +NOTE: +For more information about these migration steps, +see [Transferring your project into another namespace](../user/project/settings/index.md#transferring-an-existing-project-into-another-namespace). +A migration might result in follow-up work to update the project path in +your related resources and tools, such as websites and package managers. + +### Work with your group + +You can now view your project in your group: + +1. On the top bar, select **Menu > Groups** and find your group. +1. Look for your project under **Subgroups and projects**. + +Start enjoying the benefits of a group! For example, as the group Owner, you can +quickly view all unique users in your namespace: + +1. In your group, select **Settings > Usage Quotas**. +1. The **Seats** tab displays all users across all projects in your group. diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 903ae9c714c..d726f96f646 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -4,15 +4,7 @@ group: none info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" --- -# Deprecations by milestone - -DISCLAIMER: -This page contains information related to upcoming products, features, and functionality. -It is important to note that the information presented is for informational purposes only. -Please do not rely on this information for purchasing or planning purposes. -As with all projects, the items mentioned on this page are subject to change or delay. -The development, release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +# Deprecations by version <!-- vale off --> @@ -38,44 +30,139 @@ For deprecation reviewers (Technical Writers only): {::options parse_block_html="true" /} -View deprecations by the product versions in which they were announced. - -Each deprecation has a **planned removal milestone** and indicates whether it is a breaking change. +In each release, GitLab announces features that are deprecated and no longer recommended for use. +Each deprecated feature will be removed in a future release. +Some features cause breaking changes when they are removed. -Most of the deprecations are **planned for removal in 15.0**, and many of them are **breaking changes**. +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. <div class="js-deprecation-filters"></div> <div class="announcement-milestone"> -## 15.0 +## Announced in 15.1 + +<div class="deprecation removal-160 breaking-change"> + +### Jira GitHub Enterprise DVCS integration + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The [Jira DVCS Connector](https://docs.gitlab.com/ee/integration/jira/dvcs.html) (which enables the [Jira Development Panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/)), will no longer support Jira Cloud users starting with GitLab 16.0. The [GitLab for Jira App](https://docs.gitlab.com/ee/integration/jira/connect-app.html) has always been recommended for Jira Cloud users, and it will be required instead of the DVCS connector. If you are a Jira Cloud user, we recommended you begin migrating to the GitLab for Jira App. +Any Jira Server and Jira Data Center users will need to confirm they are not using the GitHub Enterprise Connector to enable the GitLab DVCS integration, but they may continue to use the [native GitLab DVCS integration](https://docs.gitlab.com/ee/integration/jira/dvcs.html) (supported in Jira 8.14 and later). + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### PipelineSecurityReportFinding name GraphQL field + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https://gitlab.com/gitlab-org/gitlab/-/issues/335372) to include a new `title` field. This field is an alias for the current `name` field, making the less specific `name` field redundant. The `name` field will be removed from the PipelineSecurityReportFinding type in GitLab 16.0. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### PipelineSecurityReportFinding projectFingerprint GraphQL field + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The [`project_fingerprint`](https://gitlab.com/groups/gitlab-org/-/epics/2791) attribute of vulnerability findings is being deprecated in favor of a `uuid` attribute. By using UUIDv5 values to identify findings, we can easily associate any related entity with a finding. The `project_fingerprint` attribute is no longer being used to track findings, and will be removed in GitLab 16.0. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### REST API Runner maintainer_note + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The `maintainer_note` argument in the `POST /runners` REST endpoint was deprecated in GitLab 14.8 and replaced with the `maintenance_note` argument. +The `maintainer_note` argument will be removed in GitLab 16.0. + +</div> + +<div class="deprecation removal-153"> + +### Vulnerability Report sort by Tool + +Planned removal: GitLab <span class="removal-milestone">15.3</span> (2022-08-22) + +The ability to sort the Vulnerability Report by the `Tool` column (scan type) was disabled and put behind a feature flag in GitLab 14.10 due to a refactor +of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting +by this value remains performant. Due to very low usage of the `Tool` column for sorting, the feature flag will instead be removed in +GitLab 15.3 to simplify the codebase and prevent any unwanted performance degradation. + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### project.pipeline.securityReportFindings GraphQL query + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +Previous work helped [align the vulnerabilities calls for pipeline security tabs](https://gitlab.com/gitlab-org/gitlab/-/issues/343469) to match the vulnerabilities calls for project-level and group-level vulnerability reports. This helped the frontend have a more consistent interface. The old `project.pipeline.securityReportFindings` query was formatted differently than other vulnerability data calls. Now that it has been replaced with the new `project.pipeline.vulnerabilities` field, the old `project.pipeline.securityReportFindings` is being deprecated and will be removed in GitLab 16.0. + +</div> +</div> + +<div class="announcement-milestone"> + +## Announced in 15.0 <div class="deprecation removal-160 breaking-change"> ### CiCdSettingsUpdate mutation renamed to ProjectCiCdSettingsUpdate +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `CiCdSettingsUpdate` mutation was renamed to `ProjectCiCdSettingsUpdate` in GitLab 15.0. The `CiCdSettingsUpdate` mutation will be removed in GitLab 16.0. Any user scripts that use the `CiCdSettingsUpdate` mutation must be updated to use `ProjectCiCdSettingsUpdate` instead. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### GraphQL API legacyMode argument for Runner status +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `legacyMode` argument to the `status` field in `RunnerType` will be rendered non-functional in the 16.0 release as part of the deprecations details in the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351109). @@ -84,18 +171,17 @@ In GitLab 16.0 and later, the `status` field will act as if `legacyMode` is null be present during the 16.x cycle to avoid breaking the API signature, and will be removed altogether in the 17.0 release. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### PostgreSQL 12 deprecated +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Support for PostgreSQL 12 is scheduled for removal in GitLab 16.0. In GitLab 16.0, PostgreSQL 13 becomes the minimum required PostgreSQL version. @@ -105,50 +191,49 @@ PostgreSQL 13 will also be supported for instances that want to upgrade prior to Upgrading to PostgreSQL 13 is not yet supported for GitLab instances with Geo enabled. Geo support for PostgreSQL 13 will be announced in a minor release version of GitLab 15, after the process is fully supported and validated. For more information, read the Geo related verifications on the [support epic for PostgreSQL 13](https://gitlab.com/groups/gitlab-org/-/epics/3832). -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-152"> ### Vulnerability Report sort by State +Planned removal: GitLab <span class="removal-milestone">15.2</span> (2022-07-22) + The ability to sort the Vulnerability Report by the `State` column was disabled and put behind a feature flag in GitLab 14.10 due to a refactor of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed in GitLab 15.2 to simplify the codebase and prevent any unwanted performance degradation. -**Planned removal milestone: <span class="removal-milestone">15.2</span> (2022-07-22)** </div> </div> <div class="announcement-milestone"> -## 14.10 +## Announced in 14.10 <div class="deprecation removal-150 breaking-change"> ### Dependency Scanning default Java version changed to 17 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 15.0, for Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_Java_Version` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### Manual iteration management +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Manual iteration management is deprecated and only automatic iteration cadences will be supported in the future. @@ -170,55 +255,52 @@ arguments will be removed: For more information about iteration cadences, you can refer to [the documentation of the feature](https://docs.gitlab.com/ee/user/group/iterations/#iteration-cadences). -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Outdated indices of Advanced Search migrations +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As Advanced Search migrations usually require support multiple code paths for a long period of time, it’s important to clean those up when we safely can. We use GitLab major version upgrades as a safe time to remove backward compatibility for indices that have not been fully migrated. See the [upgrade documentation](https://docs.gitlab.com/ee/update/index.html#upgrading-to-a-new-major-version) for details. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### Toggle notes confidentiality on APIs +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. Updating notes confidential attribute is no longer supported by any means. We are changing this to simplify the experience and prevent private information from being unintentionally exposed. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.9 +## Announced in 14.9 <div class="deprecation removal-150 breaking-change"> ### Background upload for object storage +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. -To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` to upload files is deprecated and will be fully removed in GitLab 15.0. +To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` to upload files is deprecated and will be fully removed in GitLab 15.0. Review the [15.0 specific changes](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html) for the [removed background uploads settings for object storage](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#removed-background-uploads-settings-for-object-storage). This impacts a small subset of object storage providers: @@ -227,51 +309,51 @@ This impacts a small subset of object storage providers: GitLab will publish additional guidance to assist affected customers in migrating. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-151"> ### Deprecate support for Debian 9 +Planned removal: GitLab <span class="removal-milestone">15.1</span> (2022-06-22) + Long term service and support (LTSS) for [Debian 9 Stretch ends in July 2022](https://wiki.debian.org/LTS). Therefore, we will no longer support the Debian 9 distribution for the GitLab package. Users can upgrade to Debian 10 or Debian 11. -**Planned removal milestone: <span class="removal-milestone">15.1</span> (2022-06-22)** </div> <div class="deprecation removal-150"> ### GitLab Pages running as daemon +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + In 15.0, support for daemon mode for GitLab Pages will be removed. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### GitLab self-monitoring project +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab self-monitoring gives administrators of self-hosted GitLab instances the tools to monitor the health of their instances. This feature is deprecated in GitLab 14.9, and is scheduled for removal in 16.0. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### GraphQL permissions change for Package settings +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. @@ -282,95 +364,93 @@ The permissions model for GraphQL is being updated. After 15.0, users with the G - [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) - [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting) -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and will be removed from GitLab Shell in GitLab 15.0. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-1410 breaking-change"> ### Permissions change for downloading Composer dependencies +Planned removal: GitLab <span class="removal-milestone">14.10</span> (2022-04-22) + WARNING: -This feature will be changed or removed in 14.10 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. -**Planned removal milestone: <span class="removal-milestone">14.10</span> (2022-04-22)** </div> <div class="deprecation removal-150 breaking-change"> ### htpasswd Authentication for the Container Registry +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### user_email_lookup_limit API field +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `user_email_lookup_limit` [API field](https://docs.gitlab.com/ee/api/settings.html) is deprecated and will be removed in GitLab 15.0. Until GitLab 15.0, `user_email_lookup_limit` is aliased to `search_rate_limit` and existing workflows will continue to work. Any API calls attempting to change the rate limits for `user_email_lookup_limit` should use `search_rate_limit` instead. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.8 +## Announced in 14.8 <div class="deprecation removal-149"> ### Configurable Gitaly `per_repository` election strategy +Planned removal: GitLab <span class="removal-milestone">14.9</span> (2022-03-22) + Configuring the `per_repository` Gitaly election strategy is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352612). `per_repository` has been the only option since GitLab 14.0. This change is part of regular maintenance to keep our codebase clean. -**Planned removal milestone: <span class="removal-milestone">14.9</span> (2022-03-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Container Network and Host Security +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). @@ -383,18 +463,17 @@ As part of this change, the following specific capabilities within GitLab are no For additional context, or to provide feedback regarding this change, please reference our open [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7476). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Dependency Scanning Python 3.9 and 3.6 image deprecation +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. For those using Dependency Scanning for Python projects, we are deprecating the default `gemnasium-python:2` image which uses Python 3.6 as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). @@ -416,22 +495,24 @@ gemnasium-python-dependency_scanning: name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 ``` -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2021-05-22)** </div> <div class="deprecation removal-150"> ### Deprecate Geo Admin UI Routes +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + In GitLab 13.0, we introduced new project and design replication details routes in the Geo Admin UI. These routes are `/admin/geo/replication/projects` and `/admin/geo/replication/designs`. We kept the legacy routes and redirected them to the new routes. In GitLab 15.0, we will remove support for the legacy routes `/admin/geo/projects` and `/admin/geo/designs`. Please update any bookmarks or scripts that may use the legacy routes. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Deprecate custom Geo:db:* Rake tasks +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + In GitLab 14.8, we are [replacing the `geo:db:*` Rake tasks with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) that are now possible after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458). The following `geo:db:*` tasks will be replaced with their corresponding `db:*:geo` tasks: @@ -453,33 +534,31 @@ The following `geo:db:*` tasks will be replaced with their corresponding `db:*:g - `geo:db:test:load` -> `db:test:load:geo` - `geo:db:test:purge` -> `db:test:purge:geo` -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Deprecate feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede CODEOWNERS. The CODEOWNERS feature will no longer be available for access control. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Deprecate legacy Gitaly configuration methods +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). These variables are being replaced with standard [`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). @@ -487,18 +566,17 @@ These variables are being replaced with standard [`config.toml` Gitaly configura GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using `config.toml`. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Elasticsearch 6.8 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Elasticsearch 6.8 is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Customers using Elasticsearch 6.8 need to upgrade their Elasticsearch version to 7.x prior to upgrading to GitLab 15.0. @@ -506,18 +584,17 @@ We recommend using the latest version of Elasticsearch 7 to benefit from all Ela Elasticsearch 6.8 is also incompatible with Amazon OpenSearch, which we [plan to support in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/327560). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### External status check API breaking changes +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to support pass-by-default requests to mark a status check as passing. Pass-by-default requests are now deprecated. @@ -535,18 +612,17 @@ and set to `passed`. Requests that: To align with this change, API calls to list external status checks will also return the value of `passed` rather than `approved` for status checks that have passed. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### GraphQL ID and GlobalID compatibility +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-04-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We are removing a non-standard extension to our GraphQL processor, which we added for backwards compatibility. This extension modifies the validation of GraphQL queries, allowing the use of the `ID` type for arguments where it would normally be rejected. Some arguments originally had the type `ID`. These were changed to specific @@ -600,18 +676,17 @@ You should convert any queries in the first form (using `ID` as a named type in to one of the other two forms (using the correct appropriate type in the signature, or using an inline argument expression). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-04-22)** </div> <div class="deprecation removal-150 breaking-change"> ### OAuth tokens without expiration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens had no expiration. In GitLab 15.0, an expiry will be automatically generated for any existing token that does not @@ -623,52 +698,49 @@ tokens before GitLab 15.0 is released: 1. Edit the application. 1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Optional enforcement of PAT expiration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The feature to disable enforcement of PAT expiration is unusual from a security perspective. We have become concerned that this unusual feature could create unexpected behavior for users. Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Optional enforcement of SSH expiration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The feature to disable enforcement of SSH expiration is unusual from a security perspective. We have become concerned that this unusual feature could create unexpected behavior for users. Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Out-of-the-box SAST support for Java 8 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [GitLab SAST SpotBugs analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) scans [Java, Scala, Groovy, and Kotlin code](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) for security vulnerabilities. For technical reasons, the analyzer must first compile the code before scanning. @@ -683,33 +755,31 @@ In GitLab 15.0, we will: If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Querying Usage Trends via the `instanceStatisticsMeasurements` GraphQL node +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTrendsMeasurements` in 13.10 and the old field name has been marked as deprecated. To fix the existing GraphQL queries, replace `instanceStatisticsMeasurements` with `usageTrendsMeasurements`. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### REST API Runner will not accept `status` filter values of `active` or `paused` +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Runner REST endpoints will stop accepting `paused` or `active` as a status value in GitLab 16.0. @@ -719,33 +789,31 @@ Status values `paused` or `active` will no longer be accepted and will be replac When checking for paused runners, API users are advised to specify `paused=true` as the query parameter. When checking for active runners, specify `paused=false`. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** </div> <div class="deprecation removal-160 breaking-change"> ### REST API endpoint to list group runners no longer accepts `project_type` value for `type` argument +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `GET /groups/:id/runners?type=project_type` endpoint will be removed in GitLab 16.0. The endpoint always returned an empty collection. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** </div> <div class="deprecation removal-160 breaking-change"> ### REST and GraphQL API Runner usage of `active` replaced by `paused` +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Occurrences of the `active` identifier in the GitLab Runner REST and GraphQL API endpoints will be renamed to `paused` in GitLab 16.0, namely: @@ -767,18 +835,17 @@ The 16.0 release of the GitLab Runner will start using the `paused` property whe will only be compatible with GitLab 16.0 and later. Until 16.0, GitLab will accept the deprecated `active` flag from existing runners. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Request profiling +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. [Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/request_profiling.html) is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. @@ -788,18 +855,17 @@ It also depends on a few third-party gems that are not actively maintained anymo For more information, check the [summary section of the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488#deprecation-summary). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Required pipeline configurations in Premium tier +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) feature is deprecated in GitLab 14.8 for Premium customers and is scheduled for removal in GitLab 15.0. This feature is not deprecated for GitLab Ultimate customers. @@ -808,35 +874,33 @@ This change to move the feature to GitLab's Ultimate tier is intended to help ou This change will also help GitLab remain consistent in its tiering strategy with the other related Ultimate-tier features of: [Security policies](https://docs.gitlab.com/ee/user/application_security/policies/) and [compliance framework pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Retire-JS Dependency Scanning tool +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As of 14.8 the retire.js job is being deprecated from Dependency Scanning. It will continue to be included in our CI/CD template while deprecated. We are removing retire.js from Dependency Scanning on May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. If you have explicitly excluded retire.js using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration related to the `retire-js-dependency_scanning` job you will want to switch to gemnasium-dependency_scanning before the removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference retire.js, or customized your template specifically for retire.js, you will not need to take action. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-152 breaking-change"> ### SAST analyzer consolidation and CI/CD template changes +Planned removal: GitLab <span class="removal-milestone">15.2</span> (2022-07-22) + WARNING: -This feature will be changed or removed in 15.2 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. @@ -862,18 +926,17 @@ This change will be reflected in the automatic language detection portion of the If you applied customizations to any of the affected analyzers or if you currently disable the Semgrep analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352554#breaking-change). -**Planned removal milestone: <span class="removal-milestone">15.2</span> (2022-07-22)** </div> <div class="deprecation removal-150 breaking-change"> ### SAST support for .NET 2.1 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab SAST Security Code Scan analyzer scans .NET code for security vulnerabilities. For technical reasons, the analyzer must first build the code to scan it. @@ -895,13 +958,14 @@ Version 3 was [announced in GitLab 14.6](https://about.gitlab.com/releases/2021/ If you rely on .NET 2.1 support being present in the analyzer image by default, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352553#breaking-change). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Secret Detection configuration variables deprecated +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + To make it simpler and more reliable to [customize GitLab Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings), we're deprecating some of the variables that you could previously set in your CI/CD configuration. The following variables currently allow you to customize the options for historical scanning, but interact poorly with the [GitLab-managed CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) and are now deprecated: @@ -919,18 +983,17 @@ You'll still be able to configure historical scanning of your commit history by For further details, see [the deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352565). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Secure and Protect analyzer images published in new location +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to [scan for security vulnerabilities](https://docs.gitlab.com/ee/user/application_security/). Each analyzer is distributed as a container image. @@ -948,25 +1011,24 @@ Otherwise, you won't receive further updates. See the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564) for more details. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Secure and Protect analyzer major version update +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The Secure and Protect stages will be bumping the major versions of their analyzers in tandem with the GitLab 15.0 release. This major bump will enable a clear delineation for analyzers, between: - Those released prior to May 22, 2022, which generate reports that _are not_ subject to stringent schema validation. - Those released after May 22, 2022, which generate reports that _are_ subject to stringent schema validation. -If you are not using the default inclusion templates, or have pinned your analyzer version(s) you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. +If you are not using the default inclusion templates, or have pinned your analyzer versions you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. Users of GitLab 12.0-14.10 will continue to experience analyzer updates as normal until the release of GitLab 15.0, following which all newly fixed bugs and newly released features in the new major versions of the analyzers will not be available in the deprecated versions because we do not backport bugs and new features as per our [maintenance policy](https://docs.gitlab.com/ee/policy/maintenance.html). As required security patches will be backported within the latest 3 minor releases. Specifically, the following are being deprecated and will no longer be updated after 15.0 GitLab release: @@ -994,18 +1056,17 @@ Specifically, the following are being deprecated and will no longer be updated a - `sobelow`: version 2 - `spotbugs`: version 2 -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Support for gRPC-aware proxy deployed between Gitaly and rest of GitLab +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Although not recommended or documented, it was possible to deploy a gRPC-aware proxy between Gitaly and the rest of GitLab. For example, NGINX and Envoy. The ability to deploy a gRPC-aware proxy is @@ -1019,18 +1080,17 @@ By sending some of our internal RPC traffic through a custom protocol (instead o increase throughput and reduce Go garbage collection latency. For more information, see the [relevant epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/463). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Test coverage project CI/CD setting +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To simplify setting a test coverage pattern, in GitLab 15.0 the [project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-using-project-settings-removed) @@ -1039,18 +1099,17 @@ is being removed. Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set testing coverage results in merge requests. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Vulnerability Check +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The vulnerability check feature is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. @@ -1061,18 +1120,17 @@ The new security approvals feature is similar to vulnerability check. For exampl - A two-step approval process can be enforced for any desired changes to security approval rules. - A single set of security policies can be applied to multiple development projects to allow for ease in maintaining a single, centralized ruleset. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### `CI_BUILD_*` predefined variables +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in GitLab 9.0, and will be removed in GitLab 16.0. If you still use these variables, be sure to change to the replacement [predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) which are functionally identical: @@ -1091,66 +1149,50 @@ The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in G | `CI_BUILD_TOKEN` | `CI_JOB_TOKEN` | | `CI_BUILD_TRIGGERED` | `CI_PIPELINE_TRIGGERED` | -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** -</div> - -<div class="deprecation removal-150"> - -### `fixup!` commit messages setting draft status of associated Merge Request - -The use of `fixup!` as a commit message to trigger draft status -of the associated Merge Request is generally unused, and can cause -confusion with other uses of the term. "Draft" is the preferred -and supported trigger for triggering draft status from commit -messages, as part of our streamlining of the feature. -Support for `fixup!` is now considered deprecated, and will be -removed in GitLab 15.0. - -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-06-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `projectFingerprint` in `PipelineSecurityReportFinding` GraphQL +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `projectFingerprint` field in the [PipelineSecurityReportFinding](https://docs.gitlab.com/ee/api/graphql/reference/index.html#pipelinesecurityreportfinding) GraphQL object is being deprecated. This field contains a "fingerprint" of security findings used to determine uniqueness. The method for calculating fingerprints has changed, resulting in different values. Going forward, the new values will be exposed in the UUID field. Data previously available in the projectFingerprint field will eventually be removed entirely. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `started` iterations API field +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `started` field in the [iterations API](https://docs.gitlab.com/ee/api/iterations.html#list-project-iterations) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `current` field (already available) which aligns with the naming for other time-based entities, such as milestones. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.7 +## Announced in 14.7 <div class="deprecation removal-150"> ### Container scanning schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [Container scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported in GitLab 15.0. @@ -1162,13 +1204,14 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Coverage guided fuzzing schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [Coverage guided fuzzing report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) below version 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported in GitLab 15.0. @@ -1183,13 +1226,14 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### DAST schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [DAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported as of GitLab 15.0. @@ -1204,13 +1248,14 @@ To help with the transition, from GitLab 14.10, non-compliant reports will cause [warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Dependency scanning schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [Dependency scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported as of GitLab 15.0. @@ -1225,13 +1270,14 @@ To help with the transition, from GitLab 14.10, non-compliant reports will cause [warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Enforced validation of security report schemas +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [Security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported in GitLab 15.0. @@ -1246,68 +1292,69 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Godep support in License Compliance +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + The Godep dependency manager for Golang was deprecated in 2020 by Go and has been replaced with Go modules. To reduce our maintenance cost we are deprecating License Compliance for Godep projects as of 14.7 and will remove it in GitLab 15.0 -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Logging in GitLab +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users can search for relevant logs in GitLab. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, we don't have a recommended solution for logging within GitLab. For more information, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### Monitor performance metrics through Prometheus +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. By displaying data stored in a Prometheus instance, GitLab allows users to view performance metrics. GitLab also displays visualizations of these metrics in dashboards. The user can connect to a previously-configured external Prometheus instance, or set up Prometheus as a GitLab Managed App. However, since certificate-based integration with Kubernetes clusters is deprecated in GitLab, the metrics functionality in GitLab that relies on Prometheus is also deprecated. This includes the metrics visualizations in dashboards. GitLab is working to develop a single user experience based on [Opstrace](https://about.gitlab.com/press/releases/2021-12-14-gitlab-acquires-opstrace-to-expand-its-devops-platform-with-open-source-observability-solution.html). An [issue exists](https://gitlab.com/groups/gitlab-org/-/epics/6976) for you to follow work on the Opstrace integration. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-150"> ### Pseudonymizer +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + The Pseudonymizer feature is generally unused, can cause production issues with large databases, and can interfere with object storage development. It is now considered deprecated, and will be removed in GitLab 15.0. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### SAST schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [SAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported as of GitLab 15.0. @@ -1322,13 +1369,14 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Secret detection schemas below 14.0.0 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + [Secret detection report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation against the schema version declared in the report will also no longer be supported as of GitLab 15.0. @@ -1343,18 +1391,17 @@ To help with the transition, from GitLab 14.10, non-compliant reports will displ [warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) in the Vulnerability Report. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Sidekiq metrics and health checks configuration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Exporting Sidekiq metrics and health checks using a single process and port is deprecated. Support will be removed in 15.0. @@ -1376,162 +1423,156 @@ and only run one server (not changing the current behaviour). Only if they are both set and a different port is provided, a separate metrics server will spin up to serve the Sidekiq metrics, similar to the way Sidekiq will behave in 15.0. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### Static Site Editor +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + The Static Site Editor will no longer be available starting in GitLab 15.0. Improvements to the Markdown editing experience across GitLab will deliver smiliar benefit but with a wider reach. Incoming requests to the Static Site Editor will be redirected to the [Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/index.html). Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/static_site_editor/) for more information, including how to remove the configuration files from existing projects. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Tracing in GitLab +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users can navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab is deprecated in GitLab 14.7, and scheduled for removal in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150"> ### `artifacts:report:cobertura` keyword +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + Currently, test coverage visualizations in GitLab only support Cobertura reports. Starting 15.0, the `artifacts:report:cobertura` keyword will be replaced by [`artifacts:reports:coverage_report`](https://gitlab.com/gitlab-org/gitlab/-/issues/344533). Cobertura will be the only supported report file in 15.0, but this is the first step towards GitLab supporting other report types. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### merged_by API field +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `merged_by` field in the [merge request API](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `merge_user` field (already present in GraphQL) which more correctly identifies who merged a merge request when performing actions (merge when pipeline succeeds, add to merge train) other than a simple merge. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.6 +## Announced in 14.6 <div class="deprecation removal-150 breaking-change"> ### CI/CD job name length limit +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 15.0 we are going to limit the number of characters in CI/CD job names to 255. Any pipeline with job names that exceed the 255 character limit will stop working after the 15.0 release. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Legacy approval status names from License Compliance API +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We deprecated legacy names for approval status of license policy (blacklisted, approved) in the `managed_licenses` API but they are still used in our API queries and responses. They will be removed in 15.0. If you are using our License Compliance API you should stop using the `approved` and `blacklisted` query parameters, they are now `allowed` and `denied`. In 15.0 the responses will also stop using `approved` and `blacklisted` so you need to adjust any of your custom tools to use the old and new values so they do not break with the 15.0 release. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `type` and `types` keyword in CI/CD configuration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines that use these keywords will stop working, so you must switch to `stage` and `stages`, which have the same behavior. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### apiFuzzingCiConfigurationCreate GraphQL mutation +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The API Fuzzing configuration snippet is now being generated client-side and does not require an API request anymore. We are therefore deprecating the `apiFuzzingCiConfigurationCreate` mutation which isn't being used in GitLab anymore. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### bundler-audit Dependency Scanning tool +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As of 14.6 bundler-audit is being deprecated from Dependency Scanning. It will continue to be in our CI/CD template while deprecated. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. If you have explicitly excluded bundler-audit using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration, for example to edit the `bundler-audit-dependency_scanning` job, you will want to switch to gemnasium-dependency_scanning before removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference bundler-audit, or customized your template specifically for bundler-audit, you will not need to take action. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.5 +## Announced in 14.5 <div class="deprecation removal-150 breaking-change"> ### Changing an instance (shared) runner to a project (specific) runner +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 15.0, you can no longer change an instance (shared) runner to a project (specific) runner. @@ -1539,69 +1580,49 @@ Users often accidentally change instance runners to project runners, and they're Administrators who need to add runners for multiple projects can register a runner for one project, then go to the Admin view and choose additional projects. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Known host required for GitLab Runner SSH executor +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** -</div> - -<div class="deprecation removal-150 breaking-change"> - -### Must explicitly assign `AuthenticationType` for `[runners.cache.s3]` - -WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -In GitLab 15.0 and later, to access the AWS S3 cache, you must specify the `AuthenticationType` for [`[runners.cache.s3]`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscaches3-section). The `AuthenticationType` must be `IAM` or `credentials`. - -Prior to 14.5, if you did not define the `AuthenticationType`, GitLab Runner chose a type for you. - -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### Package pipelines in API payload is paginated +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. A request to the API for `/api/v4/projects/:id/packages` returns a paginated result of packages. Each package lists all of its pipelines in this response. This is a performance concern, as it's possible for a package to have hundreds or thousands of associated pipelines. In milestone 16.0, we will remove the `pipelines` attribute from the API response. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-160 breaking-change"> ### REST and GraphQL API Runner status will not return `paused` +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Runner REST and GraphQL API endpoints will not return `paused` or `active` as a status in GitLab 16.0. @@ -1611,18 +1632,17 @@ A runner's status will only relate to runner contact status, such as: When checking if a runner is `paused`, API users are advised to check the boolean attribute `paused` to be `true` instead. When checking if a runner is `active`, check if `paused` is `false`. -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-04-22)** </div> <div class="deprecation removal-156 breaking-change"> ### SaaS certificate-based integration with Kubernetes +Planned removal: GitLab <span class="removal-milestone">15.6</span> (2022-11-22) + WARNING: -This feature will be changed or removed in 15.6 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The certificate-based integration with Kubernetes will be [deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). As a GitLab SaaS customer, on new namespaces, you will no longer be able to integrate GitLab and your cluster using the certificate-based approach as of GitLab 15.0. The integration for current users will be enabled per namespace. The integrations are expected to be switched off completely on GitLab SaaS around 2022 November 22. @@ -1633,22 +1653,21 @@ For updates and details about this deprecation, follow [this epic](https://gitla GitLab self-managed customers can still use the feature [with a feature flag](https://docs.gitlab.com/ee/update/deprecations.html#self-managed-certificate-based-integration-with-kubernetes). -**Planned removal milestone: <span class="removal-milestone">15.6</span> (2022-11-22)** </div> <div class="deprecation removal-160 breaking-change"> ### Self-managed certificate-based integration with Kubernetes +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + WARNING: -This feature will be changed or removed in 16.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The certificate-based integration with Kubernetes [will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). -As a self-managed customer, we are introducing a feature flag in GitLab 15.0 so you can keep your certificate-based integration enabled. However, the feature flag will be disabled by default, so this change is a **breaking change**. +As a self-managed customer, we are introducing the [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) `certificate_based_clusters` in GitLab 15.0 so you can keep your certificate-based integration enabled. However, the feature flag will be disabled by default, so this change is a **breaking change**. In GitLab 16.0 we will remove both the feature and its related code. Until the final removal in 16.0, features built on this integration will continue to work, if you enable the feature flag. Until the feature is removed, GitLab will continue to fix security and critical issues as they arise. @@ -1657,116 +1676,109 @@ For a more robust, secure, forthcoming, and reliable integration with Kubernetes For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). -**Planned removal milestone: <span class="removal-milestone">16.0</span> (2023-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Support for SLES 12 SP2 +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Update to the Container Registry group-level API +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Value Stream Analytics filtering calculation change +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `Versions` on base `PackageType` +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). In milestone 15.0, we will completely remove `Version` from `PackageType`. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `defaultMergeCommitMessageWithDescription` GraphQL API field +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GraphQL API field `defaultMergeCommitMessageWithDescription` has been deprecated and will be removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `dependency_proxy_for_private_groups` feature flag +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We added a feature flag because [GitLab-#11582](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) changed how public groups use the Dependency Proxy. Prior to this change, you could use the Dependency Proxy without authentication. The change requires authentication to use the Dependency Proxy. In milestone 15.0, we will remove the feature flag entirely. Moving forward, you must authenticate when using the Dependency Proxy. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `pipelines` field from the `version` field +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: @@ -1775,64 +1787,62 @@ In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDeta To mitigate possible performance problems, we will remove the `versions` field's `pipelines` field in milestone 15.0. Although you will no longer be able to get all pipelines for all versions of a package, you can still get the pipelines of a single version through the remaining `pipelines` field for that version. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `promote-db` command from `gitlab-ctl` +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. `gitlab-ctl promote-db` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### `promote-to-primary-node` command from `gitlab-ctl` +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-148"> ### openSUSE Leap 15.2 packages +Planned removal: GitLab <span class="removal-milestone">14.8</span> (2022-02-22) + Distribution support and security updates for openSUSE Leap 15.2 are [ending December 2021](https://en.opensuse.org/Lifetime#openSUSE_Leap). Starting in 14.5 we are providing packages for openSUSE Leap 15.3, and will stop providing packages for openSUSE Leap 15.2 in the 14.8 milestone. -**Planned removal milestone: <span class="removal-milestone">14.8</span> (2022-02-22)** </div> </div> <div class="announcement-milestone"> -## 14.3 +## Announced in 14.3 <div class="deprecation removal-150 breaking-change"> ### Audit events for repository push events +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are now deprecated and will be removed in GitLab 15.0. @@ -1840,35 +1850,33 @@ These events have always been disabled by default and had to be manually enabled feature flag. Enabling them can cause too many events to be generated which can dramatically slow down GitLab instances. For this reason, they are being removed. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### GitLab Serverless +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. [GitLab Serverless](https://docs.gitlab.com/ee/user/project/clusters/serverless/) is a feature set to support Knative-based serverless development with automatic deployments and monitoring. We decided to remove the GitLab Serverless features as they never really resonated with our users. Besides, given the continuous development of Kubernetes and Knative, our current implementations do not even work with recent versions. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### Legacy database configuration +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html) configuration located in `database.yml` is changing and the legacy format is deprecated. The legacy format @@ -1876,18 +1884,17 @@ supported using a single PostgreSQL adapter, whereas the new format is changing This deprecation mainly impacts users compiling GitLab from source because Omnibus will handle this configuration automatically. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> <div class="deprecation removal-150 breaking-change"> ### OmniAuth Kerberos gem +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `omniauth-kerberos` gem will be removed in our next major release, GitLab 15.0. @@ -1895,43 +1902,46 @@ This gem has not been maintained and has very little usage. We therefore plan to Note that we are not deprecating the Kerberos SPNEGO integration, only the old password-based Kerberos integration. -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> <div class="announcement-milestone"> -## 14.2 +## Announced in 14.2 <div class="deprecation removal-146"> ### Release CLI distributed as a generic package +Planned removal: GitLab <span class="removal-milestone">14.6</span> (2021-12-22) + The [release-cli](https://gitlab.com/gitlab-org/release-cli) will be released as a [generic package](https://gitlab.com/gitlab-org/release-cli/-/packages) starting in GitLab 14.2. We will continue to deploy it as a binary to S3 until GitLab 14.5 and stop distributing it in S3 in GitLab 14.6. -**Planned removal milestone: <span class="removal-milestone">14.6</span> (2021-12-22)** </div> <div class="deprecation removal-145"> ### Rename Task Runner pod to Toolbox +Planned removal: GitLab <span class="removal-milestone">14.5</span> (2021-11-22) + The Task Runner pod is used to execute periodic housekeeping tasks within the GitLab application and is often confused with the GitLab Runner. Thus, [Task Runner will be renamed to Toolbox](https://gitlab.com/groups/gitlab-org/charts/-/epics/25). This will result in the rename of the sub-chart: `gitlab/task-runner` to `gitlab/toolbox`. Resulting pods will be named along the lines of `{{ .Release.Name }}-toolbox`, which will often be `gitlab-toolbox`. They will be locatable with the label `app=toolbox`. -**Planned removal milestone: <span class="removal-milestone">14.5</span> (2021-11-22)** </div> </div> <div class="announcement-milestone"> -## 14.0 +## Announced in 14.0 <div class="deprecation removal-156"> ### NFS for Git repository storage +Planned removal: GitLab <span class="removal-milestone">15.6</span> (2022-11-22) + With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for further information. Gitaly Cluster offers tremendous benefits for our customers such as: @@ -1942,21 +1952,19 @@ Gitaly Cluster offers tremendous benefits for our customers such as: We encourage customers currently using NFS for Git repositories to plan their migration by reviewing our documentation on [migrating to Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/index.html#migrate-to-gitaly-cluster). -**Planned removal milestone: <span class="removal-milestone">15.6</span> (2022-11-22)** </div> <div class="deprecation removal-150 breaking-change"> ### OAuth implicit grant +Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) + WARNING: -This feature will be changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The OAuth implicit grant authorization flow will be removed in our next major release, GitLab 15.0. Any applications that use OAuth implicit grant should switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). -**Planned removal milestone: <span class="removal-milestone">15.0</span> (2022-05-22)** </div> </div> diff --git a/doc/update/index.md b/doc/update/index.md index 779b8ccbc6a..416adb621d0 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -94,11 +94,13 @@ that can process jobs in the `background_migration` queue. ### Background migrations +#### Pending migrations + **For Omnibus installations:** ```shell sudo gitlab-rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' ``` **For installations from source:** @@ -106,7 +108,39 @@ sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigrati ```shell cd /home/git/gitlab sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::BackgroundMigration.remaining' -sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigrationJob.pending.count' +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.queued.count' +``` + +#### Failed migrations + +**For Omnibus installations:** + +For GitLab 14.0-14.9: + +```shell +sudo gitlab-rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +``` + +For GitLab 14.10 and later: + +```shell +sudo gitlab-rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' +``` + +**For installations from source:** + +For GitLab 14.0-14.9: + +```shell +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +``` + +For GitLab 14.10 and later: + +```shell +cd /home/git/gitlab +sudo -u git -H bundle exec rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' ``` ### Batched background migrations @@ -250,24 +284,36 @@ See [troubleshooting batched background migrations](../user/admin_area/monitorin ## Dealing with running CI/CD pipelines and jobs -If you upgrade your GitLab instance while the GitLab Runner is processing jobs, the trace updates fail. When GitLab is back online, the trace updates should self-heal. However, depending on the error, the GitLab Runner either retries or eventually terminates job handling. +If you upgrade your GitLab instance while the GitLab Runner is processing jobs, the trace updates fail. When GitLab is back online, the trace updates should self-heal. However, depending on the error, the GitLab Runner either retries, or eventually terminates, job handling. As for the artifacts, the GitLab Runner attempts to upload them three times, after which the job eventually fails. -To address the above two scenario's, it is advised to do the following prior to upgrading: +To address the above two scenarios, it is advised to do the following prior to upgrading: 1. Plan your maintenance. -1. Pause your runners. +1. Pause your runners or block new jobs from starting by adding following to your `/etc/gitlab/gitlab.rb`: + + ```ruby + nginx['custom_gitlab_server_config'] = "location /api/v4/jobs/request {\n deny all;\n return 503;\n}\n" + ``` + + And reconfigure GitLab with: + + ```shell + sudo gitlab-ctl reconfigure + ``` + 1. Wait until all jobs are finished. 1. Upgrade GitLab. 1. [Update GitLab Runner](https://docs.gitlab.com/runner/install/index.html) to the same version as your GitLab version. Both versions [should be the same](https://docs.gitlab.com/runner/#gitlab-runner-versions). +1. Unpause your runners and unblock new jobs from starting by reverting the previous `/etc/gitlab/gitlab.rb` change. ## Checking for pending Advanced Search migrations **(PREMIUM SELF)** -This section is only applicable if you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**. +This section is only applicable if you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**. -Major releases require all [Advanced Search migrations](../integration/elasticsearch.md#advanced-search-migrations) +Major releases require all [Advanced Search migrations](../integration/advanced_search/elasticsearch.md#advanced-search-migrations) to be finished from the most recent minor release in your current version before the major version upgrade. You can find pending migrations by running the following command: @@ -287,11 +333,11 @@ sudo -u git -H bundle exec rake gitlab:elastic:list_pending_migrations ### What do I do if my Advanced Search migrations are stuck? -See [how to retry a halted migration](../integration/elasticsearch.md#retry-a-halted-migration). +See [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). ### What do I do for the error `Elasticsearch version not compatible` -Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/elasticsearch.md#version-requirements). +Confirm that your version of Elasticsearch or OpenSearch is [compatible with your version of GitLab](../integration/advanced_search/elasticsearch.md#version-requirements). ## Upgrading without downtime @@ -314,7 +360,7 @@ A *major* upgrade requires the following steps: It's also important to ensure that any [background migrations have been fully completed](#checking-for-background-migrations-before-upgrading) before upgrading to a new major version. -If you have enabled the [Elasticsearch integration](../integration/elasticsearch.md) **(PREMIUM SELF)**, then +If you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**, then [ensure all Advanced Search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version within your current version before proceeding with the major version upgrade. @@ -333,7 +379,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab accordingly, while also consulting the [version-specific upgrade instructions](#version-specific-upgrading-instructions): -`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.9.0`](#1490) -> [latest `14.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) +`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#1410) -> [`15.0.Z`](#1500) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) The following table, while not exhaustive, shows some examples of the supported upgrade paths. @@ -341,7 +387,9 @@ Additional steps between the mentioned versions are possible. We list the minima | Target version | Your version | Supported upgrade path | Note | | -------------- | ------------ | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `14.6.2` | `13.10.2` | `13.10.2` -> `13.12.15` -> `14.0.12` -> `14.6.2` | Two intermediate versions are required: `13.12` and `14.0`, then `14.6.2`. | +| `15.1.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.4` -> `15.0.2` -> `15.1.0` | Three intermediate versions are required: `14.9` and `14.10`, `15.0`, then `15.1.0`. | +| `15.0.0` | `14.6.2` | `14.6.2` -> `14.9.5` -> `14.10.4` -> `15.0.2` | Two intermediate versions are required: `14.9` and `14.10`, then `15.0.0`. | +| `14.6.2` | `13.10.2` | `13.10.2` -> `13.12.15` -> `14.0.12` -> `14.3.6` => `14.6.2` | Three intermediate versions are required: `13.12` and `14.0`, `14.3`, then `14.6.2`. | | `14.1.8` | `13.9.2` | `13.9.2` -> `13.12.15` -> `14.0.12` -> `14.1.8` | Two intermediate versions are required: `13.12` and `14.0`, then `14.1.8`. | | `13.12.15` | `12.9.2` | `12.9.2` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.8.8` -> `13.12.15` | Four intermediate versions are required: `12.10`, `13.0`, `13.1` and `13.8.8`, then `13.12.15`. | | `13.2.10` | `11.5.0` | `11.5.0` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> `13.2.10` | Six intermediate versions are required: `11.11`, `12.0`, `12.1`, `12.10`, `13.0` and `13.1`, then `13.2.10`. | @@ -383,7 +431,7 @@ possible. ## Version-specific upgrading instructions -Each month, major, minor or patch releases of GitLab are published along with a +Each month, major, minor, or patch releases of GitLab are published along with a [release post](https://about.gitlab.com/releases/categories/releases/). You should read the release posts for all versions you're passing over. At the end of major and minor release posts, there are three sections to look for specifically: @@ -396,7 +444,7 @@ These include: - Steps you need to perform as part of an upgrade. For example [8.12](https://about.gitlab.com/releases/2016/09/22/gitlab-8-12-released/#upgrade-barometer) - required the Elasticsearch index to be recreated. Any older version of GitLab upgrading to 8.12 or higher would require this. + required the Elasticsearch index to be recreated. Any older version of GitLab upgrading to 8.12 or later would require this. - Changes to the versions of software we support such as [ceasing support for IE11 in GitLab 13](https://about.gitlab.com/releases/2020/03/22/gitlab-12-9-released/#ending-support-for-internet-explorer-11). @@ -410,43 +458,68 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 15.1.0 + +- If you run external PostgreSQL, particularly AWS RDS, + [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) + to avoid the database crashing. + ### 15.0.0 -- Elasticsearch 6.8 [is no longer supported](../integration/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version). +- Elasticsearch 6.8 [is no longer supported](../integration/advanced_search/elasticsearch.md#version-requirements). Before you upgrade to GitLab 15.0, [update Elasticsearch to any 7.x version](../integration/advanced_search/elasticsearch.md#upgrade-to-a-new-elasticsearch-major-version). +- If you run external PostgreSQL, particularly AWS RDS, + [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) + to avoid the database crashing. ### 14.10.0 -- Before upgrading to GitLab 14.10, you need to already have the latest 14.9.Z installed on your instance. +- Before upgrading to GitLab 14.10, you must already have the latest 14.9.Z installed on your instance. The upgrade to GitLab 14.10 executes a [concurrent index drop](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84308) of unneeded entries from the `ci_job_artifacts` database table. This could potentially run for multiple minutes, especially if the table has a lot of traffic and the migration is unable to acquire a lock. It is advised to let this process finish as restarting may result in data loss. +- If you run external PostgreSQL, particularly AWS RDS, + [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) + to avoid the database crashing. + +- Upgrading to patch level 14.10.3 or later might encounter a one-hour timeout due to a long running database data change, + if it was not completed while running GitLab 14.9. + + ```plaintext + FATAL: Mixlib::ShellOut::CommandTimeout: rails_migration[gitlab-rails] + (gitlab::database_migrations line 51) had an error: + [..] + Mixlib::ShellOut::CommandTimeout: Command timed out after 3600s: + ``` + + A workaround exists to [complete the data change and the upgrade manually](package/index.md#mixlibshelloutcommandtimeout-rails_migrationgitlab-rails--command-timed-out-after-3600s). + ### 14.9.0 -- Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. +- Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. These [batched background migrations](#batched-background-migrations) update whole database tables to ensure corresponding records in `namespaces` table for each record in `projects` table. After you update to 14.9.0 or a later 14.9 patch version, - [batched background migrations need to finish](#batched-background-migrations) + [batched background migrations must finish](#batched-background-migrations) before you update to a later version. If the migrations are not finished and you try to update to a later version, - you'll see an error like: + you see errors like: ```plaintext Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': ``` - + Or ```plaintext Error executing action `run` on resource 'bash[migrate gitlab-rails database]' ================================================================================ - + Mixlib::ShellOut::ShellCommandFailed ------------------------------------ - Command execution failed. STDOUT/STDERR suppressed for sensitive resource + Command execution failed. STDOUT/STDERR suppressed for sensitive resource ``` - GitLab 14.9.0 includes a @@ -461,10 +534,14 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap end ``` +- If you run external PostgreSQL, particularly AWS RDS, + [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) + to avoid the database crashing. + ### 14.8.0 -- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, please review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. - Updating to 14.8.2 or later will reset runner registration tokens for your groups and projects. +- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. + Updating to 14.8.2 or later resets runner registration tokens for your groups and projects. - The agent server for Kubernetes [is enabled by default](https://about.gitlab.com/releases/2022/02/22/gitlab-14-8-released/#the-agent-server-for-kubernetes-is-enabled-by-default) on Omnibus installations. If you run GitLab at scale, such as [the reference architectures](../administration/reference_architectures/index.md), @@ -503,12 +580,15 @@ that may remain stuck permanently in a **pending** state. [batched migration](../user/admin_area/monitoring/background_migrations.md) named `BackfillNamespaceIdForNamespaceRoute`. You can [ignore](https://gitlab.com/gitlab-org/gitlab/-/issues/357822) this. Retry it after you upgrade to version 14.9.x. +- If you run external PostgreSQL, particularly AWS RDS, + [check you have a PostgreSQL bug fix](#postgresql-segmentation-fault-issue) + to avoid the database crashing. ### 14.7.0 - See [LFS objects import and mirror issue in GitLab 14.6.0 to 14.7.2](#lfs-objects-import-and-mirror-issue-in-gitlab-1460-to-1472). -- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, please review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. - Updating to 14.7.4 or later will reset runner registration tokens for your groups and projects. +- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. + Updating to 14.7.4 or later resets runner registration tokens for your groups and projects. - GitLab 14.7 introduced a change where Gitaly expects persistent files in the `/tmp` directory. When using the `noatime` mount option on `/tmp` in a node running Gitaly, most Linux distributions run into [an issue with Git server hooks getting deleted](https://gitlab.com/gitlab-org/gitaly/-/issues/4113). @@ -521,12 +601,15 @@ that may remain stuck permanently in a **pending** state. sudo printf "x /tmp/gitaly-%s-*\n" hooks git-exec-path >/etc/tmpfiles.d/gitaly-workaround.conf ``` + This issue is fixed in GitLab 14.10 and later when using the [Gitaly runtime directory](https://docs.gitlab.com/omnibus/update/gitlab_14_changes.html#gitaly-runtime-directory) + to specify a location to store persistent files. + ### 14.6.0 - See [LFS objects import and mirror issue in GitLab 14.6.0 to 14.7.2](#lfs-objects-import-and-mirror-issue-in-gitlab-1460-to-1472). -- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, please review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. - Updating to 14.6.5 or later will reset runner registration tokens for your groups and projects. - +- If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post. + Updating to 14.6.5 or later resets runner registration tokens for your groups and projects. + ### 14.5.0 - When `make` is run, Gitaly builds are now created in `_build/bin` and no longer in the root directory of the source directory. If you @@ -535,17 +618,17 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo - Connections between Workhorse and Gitaly use the Gitaly `backchannel` protocol by default. If you deployed a gRPC proxy between Workhorse and Gitaly, Workhorse can no longer connect. As a workaround, [disable the temporary `workhorse_use_sidechannel`](../administration/feature_flags.md#enable-or-disable-the-feature) - feature flag. If you need a proxy between Workhorse and Gitaly, use a TCP proxy. If you have feedback about this change, please go to [this issue](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1301). + feature flag. If you need a proxy between Workhorse and Gitaly, use a TCP proxy. If you have feedback about this change, go to [this issue](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1301). -- In 14.1 we introduced a background migration that changes how we store merge request diff commits - in order to significantly reduce the amount of storage needed. +- In 14.1 we introduced a background migration that changes how we store merge request diff commits, + to significantly reduce the amount of storage needed. In 14.5 we introduce a set of migrations that wrap up this process by making sure that all remaining jobs over the `merge_request_diff_commits` table are completed. - These jobs will have already been processed in most cases so that no extra time is necessary during an upgrade to 14.5. + These jobs have already been processed in most cases so that no extra time is necessary during an upgrade to 14.5. However, if there are remaining jobs or you haven't already upgraded to 14.1, the deployment may take multiple hours to complete. - All merge request diff commits will automatically incorporate these changes, and there are no + All merge request diff commits automatically incorporate these changes, and there are no additional requirements to perform the upgrade. Existing data in the `merge_request_diff_commits` table remains unpacked until you run `VACUUM FULL merge_request_diff_commits`. But note that the `VACUUM FULL` operation locks and rewrites the entire `merge_request_diff_commits` table, @@ -567,10 +650,22 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo end ``` +- Upgrading to 14.5 (or later) [might encounter a one hour timeout](https://gitlab.com/gitlab-org/gitlab/-/issues/354211) + owing to a long running database data change. + + ```plaintext + FATAL: Mixlib::ShellOut::CommandTimeout: rails_migration[gitlab-rails] + (gitlab::database_migrations line 51) had an error: + [..] + Mixlib::ShellOut::CommandTimeout: Command timed out after 3600s: + ``` + + [There is a workaround to complete the data change and the upgrade manually](package/index.md#mixlibshelloutcommandtimeout-rails_migrationgitlab-rails--command-timed-out-after-3600s) + ### 14.4.4 -- For [zero-downtime upgrades](zero_downtime.md) on a GitLab cluster with separate Web and API nodes, you need to enable the `paginated_tree_graphql_query` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) _before_ upgrading GitLab Web nodes to 14.4. - This is because we [enabled `paginated_tree_graphql_query` by default in 14.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70913/diffs), so if GitLab UI is on 14.4 and its API is on 14.3, the frontend will have this feature enabled but the backend will have it disabled. This will result in the following error: +- For [zero-downtime upgrades](zero_downtime.md) on a GitLab cluster with separate Web and API nodes, you must enable the `paginated_tree_graphql_query` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) _before_ upgrading GitLab Web nodes to 14.4. + This is because we [enabled `paginated_tree_graphql_query` by default in 14.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70913/diffs), so if GitLab UI is on 14.4 and its API is on 14.3, the frontend has this feature enabled but the backend has it disabled. This results in the following error: ```shell bundle.esm.js:63 Uncaught (in promise) Error: GraphQL error: Field 'paginatedTree' doesn't exist on type 'Repository' @@ -623,6 +718,20 @@ for how to proceed. sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production ``` +- After upgrading to 14.3, ensure that all the `MigrateMergeRequestDiffCommitUsers` background + migration jobs have completed before continuing with upgrading to GitLab 14.5 or later. + This is especially important if your GitLab instance has a large + `merge_request_diff_commits` table. Any pending + `MigrateMergeRequestDiffCommitUsers` background migration jobs are + foregrounded in GitLab 14.5, and may take a long time to complete. + You can check the count of pending jobs for + `MigrateMergeRequestDiffCommitUsers` by using the PostgreSQL console (or `sudo + gitlab-psql`): + + ```sql + select count(*) from background_migration_jobs where class_name = 'MigrateMergeRequestDiffCommitUsers' and status = 0; + ``` + - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). ### 14.2.0 @@ -669,7 +778,7 @@ for how to proceed. - [Instances running 14.0.0 - 14.0.4 should not upgrade directly to GitLab 14.2 or later](#upgrading-to-later-14y-releases) but can upgrade to 14.1.Z. - It is not required for instances already running 14.0.5 (or higher) to stop at 14.1.Z. + It is not required for instances already running 14.0.5 (or later) to stop at 14.1.Z. 14.1 is included on the upgrade path for the broadest compatibility with self-managed installations, and ensure 14.0.0-14.0.4 installations do not encounter issues with [batched background migrations](#batched-background-migrations). @@ -694,18 +803,18 @@ Prerequisites: Long running batched background database migrations: - Database changes made by the upgrade to GitLab 14.0 can take hours or days to complete on larger GitLab instances. - These [batched background migrations](#batched-background-migrations) update whole database tables to mitigate primary key overflow and must be finished before upgrading to GitLab 14.2 or higher. + These [batched background migrations](#batched-background-migrations) update whole database tables to mitigate primary key overflow and must be finished before upgrading to GitLab 14.2 or later. - Due to an issue where `BatchedBackgroundMigrationWorkers` were [not working](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2785#note_614738345) for self-managed instances, a [fix was created](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65106) that requires an update to at least 14.0.5. The fix was also released in [14.1.0](#1410). After you update to 14.0.5 or a later 14.0 patch version, - [batched background migrations need to finish](#batched-background-migrations) + [batched background migrations must finish](#batched-background-migrations) before you update to a later version. If the migrations are not finished and you try to update to a later version, - you'll see an error like: + you see an error like: ```plaintext Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active': @@ -730,7 +839,7 @@ Other issues: 1. Upgrade first to either: - 14.0.5 or a later 14.0.Z patch release. - 14.1.0 or a later 14.1.Z patch release. - 1. [Batched background migrations need to finish](#batched-background-migrations) + 1. [Batched background migrations must finish](#batched-background-migrations) before you update to a later version [and may take longer than usual](#1400). ### 13.12.0 @@ -738,7 +847,7 @@ Other issues: - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). - Check the GitLab database [has no references to legacy storage](../administration/raketasks/storage.md#on-legacy-storage). - The GitLab 14.0 pre-install check will cause the package update to fail if there is unmigrated data: + The GitLab 14.0 pre-install check causes the package update to fail if unmigrated data exists: ```plaintext Checking for unmigrated data on legacy storage @@ -753,6 +862,15 @@ Other issues: - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). +- GitLab 13.11 includes a faulty background migration ([`RescheduleArtifactExpiryBackfillAgain`](https://gitlab.com/gitlab-org/gitlab/-/blob/ccc70031b843ff8cff1185988c2e472a521c2701/db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb)) + that incorrectly sets the `expire_at` column in the `ci_job_artifacts` database table. + Incorrect `expire_at` values can potentially cause data loss. + + To prevent this risk of data loss, you must remove the content of the `RescheduleArtifactExpiryBackfillAgain` + migration, which makes it a no-op migration. You can repeat the changes from the + [commit that makes the migration no-op in 14.9 and later](https://gitlab.com/gitlab-org/gitlab/-/blob/42c3dfc5a1c8181767bbb5c76e7c5fa6fefbbc2b/db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb). + For more information, see [how to disable a data migration](../development/database/deleting_migrations.md#how-to-disable-a-data-migration). + ### 13.10.0 See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). @@ -837,7 +955,7 @@ DETAIL: Key (project_id, type)=(NNN, ServiceName) is duplicated. Ruby 2.7.2 is required. GitLab does not start with Ruby 2.6.6 or older versions. -The required Git version is Git v2.29 or higher. +The required Git version is Git v2.29 or later. GitLab 13.6 includes a [background migration `BackfillJiraTrackerDeploymentType2`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46368) @@ -854,7 +972,7 @@ end ### 13.4.0 -GitLab 13.4.0 includes a background migration to [move all remaining repositories in legacy storage to hashed storage](../administration/raketasks/storage.md#migrate-to-hashed-storage). There are [known issues with this migration](https://gitlab.com/gitlab-org/gitlab/-/issues/259605) which are fixed in GitLab 13.5.4 and later. If possible, skip 13.4.0 and upgrade to 13.5.4 or higher instead. Note that the migration can take quite a while to run, depending on how many repositories must be moved. Be sure to check that all background migrations have completed before upgrading further. +GitLab 13.4.0 includes a background migration to [move all remaining repositories in legacy storage to hashed storage](../administration/raketasks/storage.md#migrate-to-hashed-storage). There are [known issues with this migration](https://gitlab.com/gitlab-org/gitlab/-/issues/259605) which are fixed in GitLab 13.5.4 and later. If possible, skip 13.4.0 and upgrade to 13.5.4 or later instead. The migration can take quite a while to run, depending on how many repositories must be moved. Be sure to check that all background migrations have completed before upgrading further. ### 13.3.0 @@ -929,7 +1047,7 @@ If you persist your own Rack Attack initializers between upgrades, you might - [GitLab 13.0 requires PostgreSQL 11](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab). - 12.10 is the final release that shipped with PostgreSQL 9.6, 10, and 11. - - You should make sure that your database is PostgreSQL 11 on GitLab 12.10 before upgrading to 13.0. This will require downtime. + - You should make sure that your database is PostgreSQL 11 on GitLab 12.10 before upgrading to 13.0. This upgrade requires downtime. ### 12.2.0 @@ -969,7 +1087,7 @@ for more information. When [Maintenance mode](../administration/maintenance_mode/index.md) is enabled, users cannot sign in with SSO, SAML, or LDAP. -Users who were signed in before Maintenance mode was enabled will continue to be signed in. If the administrator who enabled Maintenance mode loses their session, then they will not be able to disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/#disable-maintenance-mode). +Users who were signed in before Maintenance mode was enabled, continue to be signed in. If the administrator who enabled Maintenance mode loses their session, then they can't disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/#disable-maintenance-mode). [This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/329261) was fixed in GitLab 14.5.0 and backported into 14.4.3 and 14.3.5. @@ -979,6 +1097,20 @@ When Geo is enabled, LFS objects fail to be saved for imported or mirrored proje [This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/352368) was fixed in GitLab 14.8.0 and backported into 14.7.3. +### PostgreSQL segmentation fault issue + +If you run GitLab with external PostgreSQL, particularly AWS RDS, ensure you upgrade PostgreSQL +to patch levels to a minimum of 12.10 or 13.3 before upgrading to GitLab 14.8 or later. + +[In 14.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75511) +for GitLab Enterprise Edition and [in 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87983) +for GitLab Community Edition a GitLab feature called Loose Foreign Keys was enabled. + +After it was enabled, we have had reports of unplanned PostgreSQL restarts caused +by a database engine bug that causes a segmentation fault. + +Read more [in the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/364763). + ## Miscellaneous - [MySQL to PostgreSQL](mysql_to_postgresql.md) guides you through migrating diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md index 40dc57c554b..8ac2791c82b 100644 --- a/doc/update/mysql_to_postgresql.md +++ b/doc/update/mysql_to_postgresql.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/update/package/convert_to_ee.md b/doc/update/package/convert_to_ee.md index 8a0d55e34af..34bd8e61107 100644 --- a/doc/update/package/convert_to_ee.md +++ b/doc/update/package/convert_to_ee.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/update/package/downgrade.md b/doc/update/package/downgrade.md index 81f7d089bea..7b8b4da0383 100644 --- a/doc/update/package/downgrade.md +++ b/doc/update/package/downgrade.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/update/package/index.md b/doc/update/package/index.md index faca633f446..15f43f59425 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -31,6 +31,8 @@ GitLab package. upgrade is in progress. The user's web browser shows a `Deploy in progress` message or a `502` error. - For multi-node installations, see how to perform [zero downtime upgrades](../zero_downtime.md). +- Upgrades to multi-node installations can also be performed + [with downtime](../with_downtime.md). ## Version-specific changes @@ -128,7 +130,7 @@ or upgrade command: 1. Install the specific `gitlab-ee` package by using one of the following commands and replacing `<version>` with the next supported version you would like to install - (make sure to review the [upgrade path](../index.md#upgrade-paths) to confirm the + (make sure to review the [upgrade path](../index.md#upgrade-paths) to confirm the version you're installing is part of a supported path): ```shell @@ -308,3 +310,32 @@ To update the GPG key of the GitLab packages server run: curl --silent "https://packages.gitlab.com/gpg.key" | apt-key add - apt-get update ``` + +### `Mixlib::ShellOut::CommandTimeout: rails_migration[gitlab-rails] [..] Command timed out after 3600s` + +If database schema and data changes (database migrations) must take more than one hour to run, +upgrades fail with a `timed out` error: + +```plaintext +FATAL: Mixlib::ShellOut::CommandTimeout: rails_migration[gitlab-rails] (gitlab::database_migrations line 51) +had an error: Mixlib::ShellOut::CommandTimeout: bash[migrate gitlab-rails database] +(/opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/resources/rails_migration.rb line 16) +had an error: Mixlib::ShellOut::CommandTimeout: Command timed out after 3600s: +``` + +To fix this error: + +1. Run the remaining database migrations: + + ```shell + sudo gitlab-rake db:migrate + ``` + + This command may take a very long time to complete. Use `screen` or some other mechanism to ensure + the program is not interrupted if your SSH session drops. + +1. Complete the upgrade: + + ```shell + sudo gitlab-ctl reconfigure + ``` diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index a2d672e00ac..dc09f6063c8 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -107,7 +107,7 @@ sudo -u git -H make ### 8. Install/Update `gitlab-elasticsearch-indexer` **(PREMIUM SELF)** -Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch). +Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch). ### 9. Start application diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 99812e7fdb2..cdc3ec39f9a 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -157,10 +157,10 @@ version prior to upgrading the application server. If you're using Geo: -- Review [Geo upgrade documentation](../administration/geo/replication/updating_the_geo_sites.md). -- Read about the [Geo version-specific update instructions](../administration/geo/replication/version_specific_updates.md). -- Review Geo-specific steps when [updating the database](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). -- Create an upgrade and rollback plan for _each_ Geo node (primary and each secondary). +- Review [Geo upgrade documentation](../administration/geo/replication/upgrading_the_geo_sites.md). +- Read about the [Geo version-specific update instructions](../administration/geo/replication/version_specific_upgrades.md). +- Review Geo-specific steps when [upgrading the database](https://docs.gitlab.com/omnibus/settings/database.html#upgrading-a-geo-instance). +- Create an upgrade and rollback plan for _each_ Geo site (primary and each secondary). #### Runners @@ -170,7 +170,7 @@ After updating GitLab, upgrade your runners to match #### Elasticsearch After updating GitLab, you may have to upgrade -[Elasticsearch if the new version breaks compatibility](../integration/elasticsearch.md#version-requirements). +[Elasticsearch if the new version breaks compatibility](../integration/advanced_search/elasticsearch.md#version-requirements). Updating Elasticsearch is **out of scope for GitLab Support**. ## Troubleshooting diff --git a/doc/update/removals.md b/doc/update/removals.md index 4e653d5ab0a..299d1b4c341 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -4,7 +4,10 @@ group: none info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" --- -# Removals by milestone +# Removals by version + +In each release, GitLab removes features that were deprecated in an earlier release. +Some features cause breaking changes when they are removed. <!-- vale off --> @@ -28,30 +31,26 @@ For removal reviewers (Technical Writers only): https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-removals-doc --> -## 15.0 +## Removed in 15.0 ### API: `stale` status returned instead of `offline` or `not_connected` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The Runner [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints have changed in 15.0. -A breaking change was made to the Runner [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints -in 15.0. +If a runner has not contacted the GitLab instance in more than three months, the API returns `stale` instead of `offline` or `not_connected`. +The `stale` status was introduced in 14.6. -Instead of the GitLab Runner API endpoints returning `offline` and `not_connected` for runners that have not -contacted the GitLab instance in the past three months, the API endpoints now return the `stale` value, -which was introduced in 14.6. +The `not_connected` status is no longer valid. It was replaced with `never_contacted`. Available statuses are `online`, `offline`, `stale`, and `never_contacted`. ### Audit events for repository push events WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are removed as of GitLab 15.0. @@ -63,10 +62,8 @@ Please note that we will add high-volume audit events in the future as part of [ ### Background upload for object storage WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` has been removed in GitLab 15.0. @@ -80,10 +77,8 @@ If your object storage provider does not support `background_upload`, please [mi ### Container Network and Host Security WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). @@ -99,10 +94,8 @@ For additional context, or to provide feedback regarding this change, please ref ### Container registry authentication with htpasswd WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. @@ -134,40 +127,32 @@ The following `geo:db:*` tasks have been removed from GitLab 15.0 and have been ### DS_DEFAULT_ANALYZERS environment variable WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We are removing the `DS_DEFAULT_ANALYZERS` environment variable from Dependency Scanning on May 22, 2022 in 15.0. After this removal, this variable's value will be ignored. To configure which analyzers to run with the default configuration, you should use the `DS_EXCLUDED_ANALYZERS` variable instead. ### Dependency Scanning default Java version changed to 17 WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. For Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency Scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_JAVA_VERSION` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). Please note that consequently the default version of Gradle is now 7.3.3. -### ELK stack logging removed in GitLab 15.0 +### ELK stack logging WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users could search for relevant logs in GitLab directly. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, this feature is no longer available. For more information on the future of logging and observability, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). ### Elasticsearch 6.8.x in GitLab 15.0 WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Elasticsearch 6.8 support has been removed in GitLab 15.0. Elasticsearch 6.8 has reached [end of life](https://www.elastic.co/support/eol). If you use Elasticsearch 6.8, **you must upgrade your Elasticsearch version to 7.x** prior to upgrading to GitLab 15.0. @@ -178,20 +163,16 @@ View the [version requirements](https://docs.gitlab.com/ee/integration/elasticse ### End of support for Python 3.6 in Dependency Scanning WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. For those using Dependency Scanning for Python projects, we are removing support for the default `gemnasium-python:2` image which uses Python 3.6, as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). ### External status check API breaking changes WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to support pass-by-default requests to mark a status check as passing. Pass-by-default requests are now removed. @@ -212,10 +193,8 @@ To align with this change, API calls to list external status checks also return ### GitLab Serverless WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. All functionality related to GitLab Serverless was deprecated in GitLab 14.3 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to explore using the following technologies with GitLab CI/CD: @@ -227,20 +206,16 @@ For additional context, or to provide feedback regarding this change, please ref ### Gitaly nodes in virtual storage WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Configuring the Gitaly nodes directly in the virtual storage's root configuration object has been deprecated in GitLab 13.12 and is no longer supported in GitLab 15.0. You must move the Gitaly nodes under the `'nodes'` key as described in [the Praefect configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#praefect). ### GraphQL permissions change for Package settings WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. @@ -253,23 +228,19 @@ The permissions model for GraphQL is being updated. After 15.0, users with the G The issue for this removal is [GitLab-#350682](https://gitlab.com/gitlab-org/gitlab/-/issues/350682) -### Jaeger integration removed in GitLab 15.0 +### Jaeger integration WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users could previously navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab was deprecated in GitLab 14.7, and removed in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). ### Known host required for GitLab Runner SSH executor WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml`. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. @@ -286,40 +257,32 @@ We have now removed the deprecated legacy names for approval status of license p ### Move Gitaly Cluster Praefect `database_host_no_proxy` and `database_port_no_proxy configs` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The Gitaly Cluster configuration keys for `praefect['database_host_no_proxy']` and `praefect['database_port_no_proxy']` are replaced with `praefect['database_direct_host']` and `praefect['database_direct_port']`. ### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and is removed from GitLab Shell in GitLab 15.0. ### OAuth implicit grant WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The OAuth implicit grant authorization flow is no longer supported. Any applications that use OAuth implicit grant must switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). ### OAuth tokens without an expiration WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab no longer supports OAuth tokens [without an expiration](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens). @@ -328,10 +291,8 @@ Any existing token without an expiration has one automatically generated and app ### Optional enforcement of SSH expiration WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Disabling SSH expiration enforcement is unusual from a security perspective and could create unusual situations where an expired key is unintentionally able to be used. Unexpected behavior in a security feature is inherently dangerous and so now we enforce @@ -340,10 +301,8 @@ expiration on all SSH keys. ### Optional enforcement of personal access token expiration WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Allowing expired personal access tokens to be used is unusual from a security perspective and could create unusual situations where an expired key is unintentionally able to be used. Unexpected behavior in a security feature is inherently dangerous and so we now do not let expired personal access tokens be used. @@ -364,68 +323,11 @@ As of GitLab 15.0, we've: If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change). -### Pseudonymizer - -WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -The Pseudonymizer feature is generally unused, can cause production issues with large databases, and can interfere with object storage development. -It was removed in GitLab 15.0. - -### Remove Versions from PackageType - -WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). - -In GitLab 15.0, we will completely remove `Version` from `PackageType`. - -### Remove `promote-to-primary-node` command from `gitlab-ctl` - -WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` has been removed in GitLab 15.0. - -### Remove `type` and `types` keyword from CI/CD configuration - -WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -The `type` and `types` CI/CD keywords is removed in GitLab 15.0, so pipelines that use these keywords fail with a syntax error. Switch to `stage` and `stages`, which have the same behavior. - -### Remove dependency_proxy_for_private_groups feature flag +### Pipelines field from the version field WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. - -A feature flag was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 as part of the change to require authentication to use the Dependency Proxy. Before GitLab 13.7, you could use the Dependency Proxy without authentication. - -In GitLab 15.0, we will remove the feature flag, and you must always authenticate when you use the Dependency Proxy. - -### Remove pipelines field from the version field - -WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: @@ -434,23 +336,20 @@ In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDeta To mitigate possible performance problems, we will remove the `versions` field's `pipelines` field in GitLab 15.0. Although you will no longer be able to get all pipelines for all versions of a package, you can still get the pipelines of a single version through the remaining `pipelines` field for that version. -### Removed feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS +### Pseudonymizer WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. -The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` has been removed in GitLab 15.0. From now on, push rules will supersede CODEOWNERS. The CODEOWNERS feature is no longer available for access control. +The Pseudonymizer feature is generally unused, can cause production issues with large databases, and can interfere with object storage development. +It was removed in GitLab 15.0. ### Request profiling WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. [Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/request_profiling.html) has been removed in GitLab 15.0. @@ -463,10 +362,8 @@ For more information, check the [summary section of the deprecation issue](https ### Required pipeline configurations in Premium tier WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. [Required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) helps to define and mandate organization-wide pipeline configurations and is a requirement at an executive and organizational level. To align better with our [pricing philosophy](https://about.gitlab.com/company/pricing/#three-tiers), this feature is removed from the Premium tier in GitLab 15.0. This feature continues to be available in the GitLab Ultimate tier. @@ -480,10 +377,8 @@ This change also helps GitLab remain consistent in our tiering strategy with the ### Retire-JS Dependency Scanning tool WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We have removed support for retire.js from Dependency Scanning as of May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. @@ -492,10 +387,8 @@ If you have explicitly excluded retire.js using the `DS_EXCLUDED_ANALYZERS` vari ### Runner status `not_connected` API value WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Runner REST and GraphQL [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints deprecated the `not_connected` status value in GitLab 14.6 and will start returning `never_contacted` in its place @@ -527,10 +420,8 @@ If you rely on .NET 2.1 support being present in the analyzer image by default, ### SUSE Linux Enterprise Server 12 SP2 WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. @@ -555,10 +446,8 @@ For further details, see [the deprecation issue for this change](https://gitlab. ### Sidekiq configuration for metrics and health checks WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 15.0, you can no longer serve Sidekiq metrics and health checks over a single address and port. @@ -581,20 +470,16 @@ The Static Site Editor was deprecated in GitLab 14.7 and the feature is being re ### Support for `gitaly['internal_socket_dir']` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Gitaly introduced a new directory that holds all runtime data Gitaly requires to operate correctly. This new directory replaces the old internal socket directory, and consequentially the usage of `gitaly['internal_socket_dir']` was deprecated in favor of `gitaly['runtime_dir']`. -### Support for legacy format of `config/database.yml` removed +### Support for legacy format of `config/database.yml` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The syntax of [GitLab's database](https://docs.gitlab.com/omnibus/settings/database.html) configuration located in `database.yml` has changed and the legacy format has been removed. @@ -607,10 +492,8 @@ Instructions are available [in the source update documentation](https://docs.git ### Test coverage project CI/CD setting WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To specify a test coverage pattern, in GitLab 15.0 the [project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-to-a-merge-request-removed) @@ -622,32 +505,36 @@ To set test coverage parsing, use the project’s `.gitlab-ci.yml` file by provi ### The `promote-db` command is no longer available from `gitlab-ctl` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. The `gitlab-ctl promote-db` command has been removed in GitLab 15.0. ### Update to the Container Registry group-level API WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. +### Versions from `PackageType` + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). + +In GitLab 15.0, we will completely remove `Version` from `PackageType`. + ### Vulnerability Check WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The vulnerability check feature was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. @@ -661,10 +548,8 @@ The new security approvals feature is similar to vulnerability check. For exampl ### `Managed-Cluster-Applications.gitlab-ci.yml` WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `Managed-Cluster-Applications.gitlab-ci.yml` CI/CD template is being removed. If you need an alternative, try the [Cluster Management project template](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) instead. If your are not ready to move, you can copy the [last released version](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/v14.10.1/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml) of the template into your project. @@ -675,13 +560,29 @@ keyword has been [replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/344533 [`artifacts:reports:coverage_report`](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report). Cobertura is the only supported report file, but this is the first step towards GitLab supporting other report types. +### `defaultMergeCommitMessageWithDescription` GraphQL API field + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The GraphQL API field `defaultMergeCommitMessageWithDescription` has been removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template. + +### `dependency_proxy_for_private_groups` feature flag + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +A feature flag was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 as part of the change to require authentication to use the Dependency Proxy. Before GitLab 13.7, you could use the Dependency Proxy without authentication. + +In GitLab 15.0, we will remove the feature flag, and you must always authenticate when you use the Dependency Proxy. + ### `omniauth-kerberos` gem WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `omniauth-kerberos` gem is no longer supported. This gem has not been maintained and has very little usage. Therefore, we removed support for this authentication method and recommend using [SPEGNO](https://en.wikipedia.org/wiki/SPNEGO) instead. You can @@ -690,47 +591,65 @@ to upgrade from the removed integration to the new supported one. We are not removing Kerberos SPNEGO integration. We are removing the old password-based Kerberos. +### `promote-to-primary-node` command from `gitlab-ctl` + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` has been removed in GitLab 15.0. + +### `push_rules_supersede_code_owners` feature flag + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The `push_rules_supersede_code_owners` feature flag has been removed in GitLab 15.0. From now on, push rules will supersede the `CODEOWNERS` file. The code owners feature is no longer available for access control. + +### `type` and `types` keyword from CI/CD configuration + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. + +The `type` and `types` CI/CD keywords is removed in GitLab 15.0, so pipelines that use these keywords fail with a syntax error. Switch to `stage` and `stages`, which have the same behavior. + ### bundler-audit Dependency Scanning tool WARNING: -This feature was changed or removed in 15.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal, Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. If you have explicitly excluded bundler-audit using the `DS_EXCLUDED_ANALYZERS` variable, then you will be able to remove the reference to bundler-audit. If you have customized your pipeline’s Dependency Scanning configuration related to the `bundler-audit-dependency_scanning` job, then you will want to switch to `gemnasium-dependency_scanning`. If you have not used the `DS_EXCLUDED_ANALYZERS` to reference bundler-audit or customized your template specifically for bundler-audit, you will not need to take any action. -## 14.10 +## Removed in 14.10 ### Permissions change for downloading Composer dependencies WARNING: -This feature was changed or removed in 14.10 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. -## 14.9 +## Removed in 14.9 ### Integrated error tracking disabled by default WARNING: -This feature was changed or removed in 14.9 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.4, GitLab released an integrated error tracking backend that replaces Sentry. This feature caused database performance issues. In GitLab 14.9, integrated error tracking is removed from GitLab.com, and turned off by default in GitLab self-managed. While we explore the future development of this feature, please consider switching to the Sentry backend by [changing your error tracking to Sentry in your project settings](https://docs.gitlab.com/ee/operations/error_tracking.html#sentry-error-tracking). For additional background on this removal, please reference [Disable Integrated Error Tracking by Default](https://gitlab.com/groups/gitlab-org/-/epics/7580). If you have feedback please add a comment to [Feedback: Removal of Integrated Error Tracking](https://gitlab.com/gitlab-org/gitlab/-/issues/355493). -## 14.6 +## Removed in 14.6 ### Limit the number of triggered pipeline to 25K in free tier @@ -740,7 +659,7 @@ A large amount of triggered pipelines in a single project impacts the performanc The [release-cli](https://gitlab.com/gitlab-org/release-cli) will be released as a [generic package](https://gitlab.com/gitlab-org/release-cli/-/packages) starting in GitLab 14.2. We will continue to deploy it as a binary to S3 until GitLab 14.5 and stop distributing it in S3 in GitLab 14.6. -## 14.3 +## Removed in 14.3 ### Introduced limit of 50 tags for jobs @@ -756,14 +675,14 @@ The support for [`gitlab_pages['use_legacy_storage']` setting](https://docs.gitl In 14.0 we removed [`domain_config_source`](https://docs.gitlab.com/ee/administration/pages/index.html#domain-source-configuration-before-140) which had been previously deprecated, and allowed users to specify disk storage. In 14.0 we added `use_legacy_storage` as a **temporary** flag to unblock upgrades, and allow us to debug issues with our users and it was deprecated and communicated for removal in 14.3. -## 14.2 +## Removed in 14.2 ### Max job log file size of 100 MB GitLab values efficiency for all users in our wider community of contributors, so we're always working hard to make sure the application performs at a high level with a lovable UX. In GitLab 14.2, we have introduced a [job log file size limit](https://docs.gitlab.com/ee/administration/instance_limits.html#maximum-file-size-for-job-logs), set to 100 megabytes by default. Administrators of self-managed GitLab instances can customize this to any value. All jobs that exceed this limit are dropped and marked as failed, helping prevent performance impacts or over-use of resources. This ensures that everyone using GitLab has the best possible experience. -## 14.1 +## Removed in 14.1 ### Remove support for `prometheus.listen_address` and `prometheus.enable` @@ -789,15 +708,13 @@ The minimum supported browser versions are: - Chromium 84. - Microsoft Edge 84. -## 14.0 +## Removed in 14.0 ### Auto Deploy CI template v1 WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.0, we will update the [Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/stages.html#auto-deploy) CI template to the latest version. This includes new features, bug fixes, and performance improvements with a dependency on the v2 [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image). Auto Deploy CI template v1 is deprecated going forward. @@ -806,10 +723,8 @@ Since the v1 and v2 versions are not backward-compatible, your project might enc ### Breaking changes to Terraform CI template WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab 14.0 renews the Terraform CI template to the latest version. The new template is set up for the GitLab Managed Terraform state, with a dependency on the GitLab `terraform-images` image, to provide a good user experience around GitLab's Infrastructure-as-Code features. @@ -818,10 +733,8 @@ The current stable and latest templates are not compatible, and the current late ### Code Quality RuboCop support changed WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. By default, the Code Quality feature has not provided support for Ruby 2.6+ if you're using the Code Quality template. To better support the latest versions of Ruby, the default RuboCop version is updated to add support for Ruby 2.4 through 3.0. As a result, support for Ruby 2.1, 2.2, and 2.3 is removed. You can re-enable support for older versions by [customizing your configuration](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality.html#rubocop-errors). @@ -830,30 +743,24 @@ Relevant Issue: [Default `codeclimate-rubocop` engine does not support Ruby 2.6+ ### Container Scanning Engine Clair WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Clair, the default container scanning engine, was deprecated in GitLab 13.9 and is removed from GitLab 14.0 and replaced by Trivy. We advise customers who are customizing variables for their container scanning job to [follow these instructions](https://docs.gitlab.com/ee/user/application_security/container_scanning/#change-scanners) to ensure that their container scanning jobs continue to work. ### DAST default template stages WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.0, we've removed the stages defined in the current `DAST.gitlab-ci.yml` template to avoid the situation where the template overrides manual changes made by DAST users. We're making this change in response to customer issues where the stages in the template cause problems when used with customized DAST configurations. Because of this removal, `gitlab-ci.yml` configurations that do not specify a `dast` stage must be updated to include this stage. ### DAST environment variable renaming and removal WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab 13.8 renamed multiple environment variables to support their broader usage in different workflows. In GitLab 14.0, the old variables have been permanently removed and will no longer work. Any configurations using these variables must be updated to the new variable names. Any scans using these variables in GitLab 14.0 and later will fail to be configured correctly. These variables are: @@ -869,10 +776,8 @@ GitLab 13.8 renamed multiple environment variables to support their broader usag ### Default Browser Performance testing job renamed in GitLab 14.0 WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Browser Performance Testing has run in a job named `performance` by default. With the introduction of [Load Performance Testing](https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html) in GitLab 13.2, this naming could be confusing. To make it clear which job is running [Browser Performance Testing](https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html), the default job name is changed from `performance` to `browser_performance` in the template in GitLab 14.0. @@ -881,20 +786,16 @@ Relevant Issue: [Rename default Browser Performance Testing job](https://gitlab. ### Default DAST spider begins crawling at target URL WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab 14.0, DAST has removed the current method of resetting the scan to the hostname when starting to spider. Prior to GitLab 14.0, the spider would not begin at the specified target path for the URL but would instead reset the URL to begin crawling at the host root. GitLab 14.0 changes the default for the new variable `DAST_SPIDER_START_AT_HOST` to `false` to better support users' intention of beginning spidering and scanning at the specified target URL, rather than the host root URL. This change has an added benefit: scans can take less time, if the specified path does not contain links to the entire site. This enables easier scanning of smaller sections of an application, rather than crawling the entire app during every scan. ### Default branch name for new repositories now `main` WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Every Git repository has an initial branch, which is named `master` by default. It's the first branch to be created automatically when you create a new repository. Future [Git versions](https://lore.kernel.org/git/pull.656.v4.git.1593009996.gitgitgadget@gmail.com/) will change the default branch name in Git from `master` to `main`. In coordination with the Git project and the broader community, [GitLab has changed the default branch name](https://gitlab.com/gitlab-org/gitlab/-/issues/223789) for new projects on both our SaaS (GitLab.com) and self-managed offerings starting with GitLab 14.0. This will not affect existing projects. @@ -905,10 +806,8 @@ For more information, check out our [blog post](https://about.gitlab.com/blog/20 ### Dependency Scanning WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As mentioned in [13.9](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#deprecations-for-dependency-scanning) and [this blog post](https://about.gitlab.com/blog/2021/02/08/composition-analysis-14-deprecations-and-removals/) several removals for Dependency Scanning take effect. @@ -919,10 +818,8 @@ Previously, to prevent the Gemnasium analyzers to fetch the advisory database at ### Deprecated GraphQL fields WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In accordance with our [GraphQL deprecation and removal process](https://docs.gitlab.com/ee/api/graphql/#deprecation-process), the following fields that were deprecated prior to 13.7 are [fully removed in 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/267966): @@ -936,30 +833,24 @@ In accordance with our [GraphQL deprecation and removal process](https://docs.gi ### DevOps Adoption API Segments WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The first release of the DevOps Adoption report had a concept of **Segments**. Segments were [quickly removed from the report](https://gitlab.com/groups/gitlab-org/-/epics/5251) because they introduced an additional layer of complexity on top of **Groups** and **Projects**. Subsequent iterations of the DevOps Adoption report focus on comparing adoption across groups rather than segments. GitLab 14.0 removes all references to **Segments** [from the GraphQL API](https://gitlab.com/gitlab-org/gitlab/-/issues/324414) and replaces them with **Enabled groups**. ### Disk source configuration for GitLab Pages WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab Pages [API-based configuration](https://docs.gitlab.com/ee/administration/pages/#gitlab-api-based-configuration) has been available since GitLab 13.0. It replaces the unsupported `disk` source configuration removed in GitLab 14.0, which can no longer be chosen. You should stop using `disk` source configuration, and move to `gitlab` for an API-based configuration. To migrate away from the 'disk' source configuration, set `gitlab_pages['domain_config_source'] = "gitlab"` in your `/etc/gitlab/gitlab.rb` file. We recommend you migrate before updating to GitLab 14.0, to identify and troubleshoot any potential problems before upgrading. ### Experimental prefix in Sidekiq queue selector options WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab supports a [queue selector](https://docs.gitlab.com/ee/administration/operations/extra_sidekiq_processes.html#queue-selector) to run only a subset of background jobs for a given process. When it was introduced, this option had an 'experimental' prefix (`experimental_queue_selector` in Omnibus, `experimentalQueueSelector` in Helm charts). @@ -968,20 +859,16 @@ As announced in the [13.6 release post](https://about.gitlab.com/releases/2020/1 ### External Pipeline Validation Service Code Changes WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. For self-managed instances using the experimental [external pipeline validation service](https://docs.gitlab.com/ee/administration/external_pipeline_validation.html), the range of error codes GitLab accepts will be reduced. Currently, pipelines are invalidated when the validation service returns a response code from `400` to `499`. In GitLab 14.0 and later, pipelines will be invalidated for the `406: Not Accepted` response code only. ### Geo Foreign Data Wrapper settings WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As [announced in GitLab 13.3](https://about.gitlab.com/releases/2020/08/22/gitlab-13-3-released/#geo-foreign-data-wrapper-settings-deprecated), the following configuration settings in `/etc/gitlab/gitlab.rb` have been removed in 14.0: @@ -993,10 +880,8 @@ As [announced in GitLab 13.3](https://about.gitlab.com/releases/2020/08/22/gitla ### GitLab OAuth implicit grant WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab is deprecating the [OAuth 2 implicit grant flow](https://docs.gitlab.com/ee/api/oauth2.html#implicit-grant-flow) as it has been removed for [OAuth 2.1](https://oauth.net/2.1/). @@ -1005,30 +890,24 @@ Migrate your existing applications to other supported [OAuth2 flows](https://doc ### GitLab Runner helper image in GitLab.com Container Registry WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In 14.0, we are now pulling the GitLab Runner [helper image](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#helper-image) from the GitLab Container Registry instead of Docker Hub. Refer to [issue #27218](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27218) for details. ### GitLab Runner installation to ignore the `skel` directory WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab Runner 14.0, the installation process will ignore the `skel` directory by default when creating the user home directory. Refer to [issue #4845](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4845) for details. ### Gitaly Cluster SQL primary elector WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Now that Praefect supports a [primary election strategy](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#repository-specific-primary-nodes) for each repository, we have removed the `sql` election strategy. The `per_repository` election strategy is the new default, which is automatically used if no election strategy was specified. @@ -1038,10 +917,8 @@ If you had configured the `sql` election strategy, you must follow the [migratio ### Global `SAST_ANALYZER_IMAGE_TAG` in SAST CI template WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. With the maturity of GitLab Secure scanning tools, we've needed to add more granularity to our release process. Previously, GitLab shared a major version number for [all analyzers and tools](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks). This requires all tools to share a major version, and prevents the use of [semantic version numbering](https://semver.org/). In GitLab 14.0, SAST removes the `SAST_ANALYZER_IMAGE_TAG` global variable in our [managed `SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml) CI template, in favor of the analyzer job variable setting the `major.minor` tag in the SAST vendored template. @@ -1052,20 +929,16 @@ This deprecation and removal changes our [previously announced plan](https://abo ### Hardcoded `master` in CI/CD templates WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Our CI/CD templates have been updated to no longer use hard-coded references to a `master` branch. In 14.0, they all use a variable that points to your project's configured default branch instead. If your CI/CD pipeline relies on our built-in templates, verify that this change works with your current configuration. For example, if you have a `master` branch and a different default branch, the updates to the templates may cause changes to your pipeline behavior. For more information, [read the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/324131). ### Helm v2 support WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Helm v2 was [officially deprecated](https://helm.sh/blog/helm-v2-deprecation-timeline/) in November of 2020, with the `stable` repository being [de-listed from the Helm Hub](https://about.gitlab.com/blog/2020/11/09/ensure-auto-devops-work-after-helm-stable-repo/) shortly thereafter. With the release of GitLab 14.0, which will include the 5.0 release of the [GitLab Helm chart](https://docs.gitlab.com/charts/), Helm v2 will no longer be supported. @@ -1074,10 +947,8 @@ Users of the chart should [upgrade to Helm v3](https://helm.sh/docs/topics/v2_v3 ### Legacy DAST domain validation WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The legacy method of DAST Domain Validation for CI/CD scans was deprecated in GitLab 13.8, and is removed in GitLab 14.0. This method of domain validation only disallows scans if the `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` environment variable is set to `true` in the `gitlab-ci.yml` file, and a `Gitlab-DAST-Permission` header on the site is not set to `allow`. This two-step method required users to opt in to using the variable before they could opt out from using the header. @@ -1086,20 +957,16 @@ For more information, see the [removal issue](https://gitlab.com/gitlab-org/gitl ### Legacy feature flags WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Legacy feature flags became read-only in GitLab 13.4. GitLab 14.0 removes support for legacy feature flags, so you must migrate them to the [new version](https://docs.gitlab.com/ee/operations/feature_flags.html). You can do this by first taking a note (screenshot) of the legacy flag, then deleting the flag through the API or UI (you don't need to alter the code), and finally create a new Feature Flag with the same name as the legacy flag you deleted. Also, make sure the strategies and environments match the deleted flag. We created a [video tutorial](https://www.youtube.com/watch?v=CAJY2IGep7Y) to help with this migration. ### Legacy fields from DAST report WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As a part of the migration to a common report format for all of the Secure scanners in GitLab, DAST is making changes to the DAST JSON report. Certain legacy fields were deprecated in 13.8 and have been completely removed in 14.0. These fields are `@generated`, `@version`, `site`, and `spider`. This should not affect any normal DAST operation, but does affect users who consume the JSON report in an automated way and use these fields. Anyone affected by these changes, and needs these fields for business reasons, is encouraged to open a new GitLab issue and explain the need. @@ -1108,50 +975,40 @@ For more information, see [the removal issue](https://gitlab.com/gitlab-org/gitl ### Legacy storage WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As [announced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#planned-removal-of-legacy-storage-in-14.0), [legacy storage](https://docs.gitlab.com/ee/administration/repository_storage_types.html#legacy-storage) has been removed in GitLab 14.0. ### License Compliance WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In 13.0, we deprecated the License-Management CI template and renamed it License-Scanning. We have been providing backward compatibility by warning users of the old template to switch. Now in 14.0, we are completely removing the License-Management CI template. Read about it in [issue #216261](https://gitlab.com/gitlab-org/gitlab/-/issues/216261) or [this blog post](https://about.gitlab.com/blog/2021/02/08/composition-analysis-14-deprecations-and-removals/). ### Limit projects returned in `GET /groups/:id/` WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To improve performance, we are limiting the number of projects returned from the `GET /groups/:id/` API call to 100. A complete list of projects can still be retrieved with the `GET /groups/:id/projects` API call. ### Make `pwsh` the default shell for newly-registered Windows Runners WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab Runner 13.2, PowerShell Core support was added to the Shell executor. In 14.0, PowerShell Core, `pwsh` is now the default shell for newly-registered Windows runners. Windows `CMD` will still be available as a shell option for Windows runners. Refer to [issue #26419](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26419) for details. ### Migrate from `SAST_DEFAULT_ANALYZERS` to `SAST_EXCLUDED_ANALYZERS` WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Until GitLab 13.9, if you wanted to avoid running one particular GitLab SAST analyzer, you needed to remove it from the [long string of analyzers in the `SAST.gitlab-ci.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/390afc431e7ce1ac253b35beb39f19e49c746bff/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L12) and use that to set the [`SAST_DEFAULT_ANALYZERS`](https://docs.gitlab.com/ee/user/application_security/sast/#docker-images) variable in your project's CI file. If you did this, it would exclude you from future new analyzers because this string hard codes the list of analyzers to execute. We avoid this problem by inverting this variable's logic to exclude, rather than choose default analyzers. Beginning with 13.9, [we migrated](https://gitlab.com/gitlab-org/gitlab/-/blob/14fed7a33bfdbd4663d8928e46002a5ef3e3282c/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L13) to `SAST_EXCLUDED_ANALYZERS` in our `SAST.gitlab-ci.yml` file. We encourage anyone who uses a [customized SAST configuration](https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings) in their project CI file to migrate to this new variable. If you have not overridden `SAST_DEFAULT_ANALYZERS`, no action is needed. The CI/CD variable `SAST_DEFAULT_ANALYZERS` has been removed in GitLab 14.0, which released on June 22, 2021. @@ -1159,30 +1016,24 @@ Beginning with 13.9, [we migrated](https://gitlab.com/gitlab-org/gitlab/-/blob/1 ### Off peak time mode configuration for Docker Machine autoscaling WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab Runner 13.0, [issue #5069](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/5069), we introduced new timing options for the GitLab Docker Machine executor. In GitLab Runner 14.0, we have removed the old configuration option, [off peak time mode](https://docs.gitlab.com/runner/configuration/autoscale.html#off-peak-time-mode-configuration-deprecated). ### OpenSUSE Leap 15.1 WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Support for [OpenSUSE Leap 15.1 is being deprecated](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5135). Support for 15.1 will be dropped in 14.0. We are now providing support for openSUSE Leap 15.2 packages. ### PostgreSQL 11 support WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab 14.0 requires PostgreSQL 12 or later. It offers [significant improvements](https://www.postgresql.org/about/news/postgresql-12-released-1976/) to indexing, partitioning, and general performance benefits. @@ -1191,30 +1042,24 @@ Starting in GitLab 13.7, all new installations default to PostgreSQL version 12. ### Redundant timestamp field from DORA metrics API payload WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The [deployment frequency project-level API](https://docs.gitlab.com/ee/api/dora4_project_analytics.html#list-project-deployment-frequencies) endpoint has been deprecated in favor of the [DORA 4 API](https://docs.gitlab.com/ee/api/dora/metrics.html), which consolidates all the metrics under one API with the specific metric as a required field. As a result, the timestamp field, which doesn't allow adding future extensions and causes performance issues, will be removed. With the old API, an example response would be `{ "2021-03-01": 3, "date": "2021-03-01", "value": 3 }`. The first key/value (`"2021-03-01": 3`) will be removed and replaced by the last two (`"date": "2021-03-01", "value": 3`). ### Release description in the Tags API WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab 14.0 removes support for the release description in the Tags API. You can no longer add a release description when [creating a new tag](https://docs.gitlab.com/ee/api/tags.html#create-a-new-tag). You also can no longer [create](https://docs.gitlab.com/ee/api/tags.html#create-a-new-release) or [update](https://docs.gitlab.com/ee/api/tags.html#update-a-release) a release through the Tags API. Please migrate to use the [Releases API](https://docs.gitlab.com/ee/api/releases/#create-a-release) instead. ### Ruby version changed in `Ruby.gitlab-ci.yml` WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. By default, the `Ruby.gitlab-ci.yml` file has included Ruby 2.5. @@ -1225,10 +1070,8 @@ Relevant Issue: [Updates Ruby version 2.5 to 3.0](https://gitlab.com/gitlab-org/ ### SAST analyzer `SAST_GOSEC_CONFIG` variable WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. With the release of [SAST Custom Rulesets](https://docs.gitlab.com/ee/user/application_security/sast/#customize-rulesets) in GitLab 13.5 we allow greater flexibility in configuration options for our Go analyzer (GoSec). As a result we no longer plan to support our less flexible [`SAST_GOSEC_CONFIG`](https://docs.gitlab.com/ee/user/application_security/sast/#analyzer-settings) analyzer setting. This variable was deprecated in GitLab 13.10. GitLab 14.0 removes the old `SAST_GOSEC_CONFIG variable`. If you use or override `SAST_GOSEC_CONFIG` in your CI file, update your SAST CI configuration or pin to an older version of the GoSec analyzer. We strongly encourage [inheriting and overriding our managed CI templates](https://docs.gitlab.com/ee/user/application_security/sast/#overriding-sast-jobs) to future-proof your CI templates. @@ -1236,10 +1079,8 @@ GitLab 14.0 removes the old `SAST_GOSEC_CONFIG variable`. If you use or override ### Service Templates WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Service Templates are [removed in GitLab 14.0](https://gitlab.com/groups/gitlab-org/-/epics/5672). They were used to apply identical settings to a large number of projects, but they only did so at the time of project creation. @@ -1248,20 +1089,16 @@ While they solved part of the problem, _updating_ those values later proved to b ### Success and failure for finished build metric conversion WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab Runner 13.5, we introduced `failed` and `success` states for a job. To support Prometheus rules, we chose to convert `success/failure` to `finished` for the metric. In 14.0, the conversion has now been removed. Refer to [issue #26900](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26900) for details. ### Terraform template version WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. As we continuously [develop GitLab's Terraform integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/325312), to minimize customer disruption, we maintain two GitLab CI/CD templates for Terraform: @@ -1279,10 +1116,8 @@ To check the new changes, see the [new "major version" template](https://gitlab. ### Ubuntu 16.04 support WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Ubuntu 16.04 [reached end-of-life in April 2021](https://ubuntu.com/about/release-cycle), and no longer receives maintenance updates. We strongly recommend users to upgrade to a newer release, such as 20.04. @@ -1291,90 +1126,72 @@ GitLab 13.12 will be the last release with Ubuntu 16.04 support. ### Ubuntu 19.10 (Eoan Ermine) package WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. Ubuntu 19.10 (Eoan Ermine) reached end of life on Friday, July 17, 2020. In GitLab Runner 14.0, Ubuntu 19.10 (Eoan Ermine) is no longer available from our package distribution. Refer to [issue #26036](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26036) for details. ### Unicorn in GitLab self-managed WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. [Support for Unicorn](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6078) has been removed in GitLab 14.0 in favor of Puma. [Puma has a multi-threaded architecture](https://docs.gitlab.com/ee/administration/operations/puma.html) which uses less memory than a multi-process application server like Unicorn. On GitLab.com, we saw a 40% reduction in memory consumption by using Puma. ### WIP merge requests renamed 'draft merge requests' WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The WIP (work in progress) status for merge requests signaled to reviewers that the merge request in question wasn't ready to merge. We've renamed the WIP feature to **Draft**, a more inclusive and self-explanatory term. **Draft** clearly communicates the merge request in question isn't ready for review, and makes no assumptions about the progress being made toward it. **Draft** also reduces the cognitive load for new users, non-English speakers, and anyone unfamiliar with the WIP acronym. ### Web Application Firewall (WAF) WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The Web Application Firewall (WAF) was deprecated in GitLab 13.6 and is removed from GitLab 14.0. The WAF had limitations inherent in the architectural design that made it difficult to meet the requirements traditionally expected of a WAF. By removing the WAF, GitLab is able to focus on improving other areas in the product where more value can be provided to users. Users who currently rely on the WAF can continue to use the free and open source [ModSecurity](https://github.com/SpiderLabs/ModSecurity) project, which is independent from GitLab. Additional details are available in the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/271276). ### Windows Server 1903 image support WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In 14.0, we have removed Windows Server 1903. Microsoft ended support for this version on 2020-08-12. Refer to [issue #27551](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27551) for details. ### Windows Server 1909 image support WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In 14.0, we have removed Windows Server 1909. Microsoft ended support for this version on 2021-05-11. Refer to [issue #27899](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27899) for details. ### `/usr/lib/gitlab-runner` symlink from package WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In GitLab Runner 13.3, a symlink was added from `/user/lib/gitlab-runner/gitlab-runner` to `/usr/bin/gitlab-runner`. In 14.0, the symlink has been removed and the runner is now installed in `/usr/bin/gitlab-runner`. Refer to [issue #26651](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26651) for details. ### `?w=1` URL parameter to ignore whitespace changes WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To create a consistent experience for users based on their preferences, support for toggling whitespace changes via URL parameter has been removed in GitLab 14.0. ### `CI_PROJECT_CONFIG_PATH` variable WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. The `CI_PROJECT_CONFIG_PATH` [predefined project variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) has been removed in favor of `CI_CONFIG_PATH`, which is functionally the same. @@ -1385,40 +1202,32 @@ please update them to use `CI_CONFIG_PATH` instead. ### `FF_RESET_HELPER_IMAGE_ENTRYPOINT` feature flag WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In 14.0, we have deactivated the `FF_RESET_HELPER_IMAGE_ENTRYPOINT` feature flag. Refer to issue [#26679](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26679) for details. ### `FF_SHELL_EXECUTOR_USE_LEGACY_PROCESS_KILL` feature flag WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. In [GitLab Runner 13.1](https://docs.gitlab.com/runner/executors/shell.html#gitlab-131-and-later), [issue #3376](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3376), we introduced `sigterm` and then `sigkill` to a process in the Shell executor. We also introduced a new feature flag, `FF_SHELL_EXECUTOR_USE_LEGACY_PROCESS_KILL`, so you can use the previous process termination sequence. In GitLab Runner 14.0, [issue #6413](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6413), the feature flag has been removed. ### `FF_USE_GO_CLOUD_WITH_CACHE_ARCHIVER` feature flag WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab Runner 14.0 removes the `FF_USE_GO_CLOUD_WITH_CACHE_ARCHIVER` feature flag. Refer to [issue #27175](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27175) for details. ### `secret_detection_default_branch` job WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. To ensure Secret Detection was scanning both default branches and feature branches, we introduced two separate secret detection CI jobs (`secret_detection_default_branch` and `secret_detection`) in our managed [`Secret-Detection.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) template. These two CI jobs created confusion and complexity in the CI rules logic. This deprecation moves the `rule` logic into the `script` section, which then determines how the `secret_detection` job is run (historic, on a branch, commits, etc). If you override or maintain custom versions of `SAST.gitlab-ci.yml` or `Secret-Detection.gitlab-ci.yml`, you must update your CI templates. We strongly encourage [inheriting and overriding our managed CI templates](https://docs.gitlab.com/ee/user/application_security/secret_detection/#custom-settings-example) to future-proof your CI templates. GitLab 14.0 no longer supports the old `secret_detection_default_branch` job. @@ -1426,9 +1235,7 @@ If you override or maintain custom versions of `SAST.gitlab-ci.yml` or `Secret-D ### `trace` parameter in `jobs` API WARNING: -This feature was changed or removed in 14.0 -as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). -Before updating GitLab, review the details carefully to determine if you need to make any -changes to your code, settings, or workflow. +This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +Review the details carefully before upgrading. GitLab Runner was updated in GitLab 13.4 to internally stop passing the `trace` parameter to the `/api/jobs/:id` endpoint. GitLab 14.0 deprecates the `trace` parameter entirely for all other requests of this endpoint. Make sure your [GitLab Runner version matches your GitLab version](https://docs.gitlab.com/runner/#gitlab-runner-versions) to ensure consistent behavior. diff --git a/doc/update/restore_after_failure.md b/doc/update/restore_after_failure.md index cedb46a7c71..06da66088eb 100644 --- a/doc/update/restore_after_failure.md +++ b/doc/update/restore_after_failure.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index d91b3de6df1..d6548620356 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -88,7 +88,7 @@ sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production ### 4. Install `gitlab-elasticsearch-indexer` **(PREMIUM SELF)** -Please follow the [install instruction](../integration/elasticsearch.md#install-elasticsearch). +Please follow the [install instruction](../integration/advanced_search/elasticsearch.md#install-elasticsearch). ### 5. Start application diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 485dbc1b0bc..29bb956cb54 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments comments: false @@ -69,9 +69,9 @@ Download Ruby and compile it: ```shell mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz" -echo '2755b900a21235b443bb16dadd9032f784d4a88f143d852bc5d154f22b8781f1 ruby-2.7.5.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.5.tar.gz -cd ruby-2.7.5 +curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.6.tar.gz" +echo 'e7203b0cc09442ed2c08936d483f8ac140ec1c72e37bb5c401646b7866cb5d10 ruby-2.7.6.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.6.tar.gz +cd ruby-2.7.6 ./configure --disable-install-rdoc --enable-shared make @@ -107,11 +107,11 @@ Download and install Go (for Linux, 64-bit): # Remove former Go installation folder sudo rm -rf /usr/local/go -curl --remote-name --location --progress-bar "https://go.dev/dl/go1.16.10.linux-amd64.tar.gz" -echo '414cd18ce1d193769b9e97d2401ad718755ab47816e13b2a1cde203d263b55cf go1.16.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ - sudo tar -C /usr/local -xzf go1.16.10.linux-amd64.tar.gz +curl --remote-name --location --progress-bar "https://go.dev/dl/go1.17.10.linux-amd64.tar.gz" +echo '87fc728c9c731e2f74e4a999ef53cf07302d7ed3504b0839027bd9c10edaa3fd go1.17.10.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.17.10.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,gofmt} /usr/local/bin/ -rm go1.16.10.linux-amd64.tar.gz +rm go1.17.10.linux-amd64.tar.gz ``` ### 6. Update Git @@ -346,6 +346,9 @@ upgrade instructions. ### 13. Update Gitaly +If Gitaly is located on its own server, or you use Gitaly Cluster, see [Gitaly or Gitaly Cluster](zero_downtime.md#gitaly-or-gitaly-cluster) +on the Zero downtime upgrades page. + #### Compile Gitaly ```shell diff --git a/doc/update/upgrading_postgresql_using_slony.md b/doc/update/upgrading_postgresql_using_slony.md index f2bbc8d7558..d3102ca4591 100644 --- a/doc/update/upgrading_postgresql_using_slony.md +++ b/doc/update/upgrading_postgresql_using_slony.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/update/with_downtime.md b/doc/update/with_downtime.md new file mode 100644 index 00000000000..9357f70e44a --- /dev/null +++ b/doc/update/with_downtime.md @@ -0,0 +1,329 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Multi-node upgrades with downtime **(FREE SELF)** + +NOTE: +This process is a work in progress. You're welcome to provide feedback by either raising a ticket to support, +or [commenting on this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6244). + +While you can upgrade a multi-node GitLab deployment [with zero downtime](zero_downtime.md), +there are a number of constraints. In particular, you can upgrade to only one minor release +at a time, for example, from 14.6 to 14.7, then to 14.8, etc. + +If you want to upgrade to more than one minor release at a time (for example, from 14.6 to 14.9), +you need to take your GitLab instance offline, which implies downtime. +Before starting this process, verify the +[version specific upgrading instructions](index.md#version-specific-upgrading-instructions) +relevant to your [upgrade path](index.md#upgrade-paths). + +For a single node installation, you only need to [uprgade the GitLab package](package/index.md). + +The process for upgrading a number of components of a multi-node GitLab +installation is the same as for zero downtime upgrades. +The differences relate to the servers running Rails (Puma/Sidekiq) and +the order of events. + +At a high level, the process is: + +1. Shut down the GitLab application. +1. Upgrade your Consul servers. +1. Upgrade the other back-end components: + - Gitaly, Rails PostgreSQL, Redis, PgBouncer: these can be upgraded in any order. + - If you use PostgreSQL or Redis from your cloud platform and upgrades are required, + substitute the instructions for Omnibus GitLab with your cloud provider's instructions. +1. Upgrade the GitLab application (Sidekiq, Puma) and start the application up. + +If you are a Community Edition user, replace `gitlab-ee` with +`gitlab-ce` in the following commands. + +## Stop writes to the database + +Shut down Puma and Sidekiq on all servers running these processes: + +```shell +sudo gitlab-ctl stop sidekiq +sudo gitlab-ctl stop puma +``` + +## Upgrade the Consul nodes + +[Consult the Consul documentation for the complete instructions](../administration/consul.md#upgrade-the-consul-nodes). + +In summary: + +1. Check the Consul nodes are all healthy. +1. Upgrade the GitLab package on all your Consul servers: + + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee + + # Centos/RHEL + sudo yum install gitlab-ee + ``` + +1. Restart all GitLab services **one node at a time**: + + ```shell + sudo gitlab-ctl restart + ``` + +If your Consul cluster processes are not on their own servers, and are shared +with another service such as Redis HA or Patroni, ensure that you follow the +following principles when upgrading those servers: + +- Do not restart services more than one server at a time. +- Check the Consul cluster is healthy before upgrading or restarting services. + +## Upgrade the Gitaly nodes (Praefect / Gitaly Cluster) + +If you're running Gitaly cluster, follow the [zero downtime process](zero_downtime.md#gitaly-or-gitaly-cluster) +for Gitaly cluster. + +If you are using Amazon Machine Images (AMIs) on AWS, the Gitaly nodes +**should not be upgraded via the AMI process**. Gitaly nodes should **only** +be upgraded using the package upgrade. This is because: + +- Praefect tracks replicas of Git repositories by server hostname. +- Redeployment using AMIs will issue the nodes with new hostnames. +- Even though the storage will be the same, Gitaly cluster will not work after this. + +The Praefect nodes, however, can be upgraded via an AMI redeployment process: + + 1. The AMI redeployment process must include `gitlab-ctl reconfigure`. + Set `praefect['auto_migrate'] = false` on the AMI so all nodes get this. This + prevents `reconfigure` from automatically running database migrations. + 1. The first node to be redeployed with the upgraded image should be your + deploy node. + 1. After it's deployed, set `praefect['auto_migrate'] = true` in `gitlab.rb` + and apply with `gitlab-ctl reconfigure`. This will run the database + migrations. + 1. Redeploy your other Praefect nodes. + +## Upgrade the Gitaly nodes not part of Gitaly cluster + +For Gitaly servers which are not part of Gitaly cluster, update the GitLab package: + +```shell +# Debian/Ubuntu +sudo apt-get update && sudo apt-get install gitlab-ee + +# Centos/RHEL +sudo yum install gitlab-ee +``` + +If you have multiple Gitaly shards or have multiple load-balanced Gitaly nodes +using NFS, it doesn't matter in which order you upgrade the Gitaly servers. + +## Upgrade the PostgreSQL nodes + +For unclustered PostgreSQL servers: + +1. Upgrade the GitLab package: + + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee + + # Centos/RHEL + sudo yum install gitlab-ee + ``` + +1. The upgrade process does not restart PostgreSQL when the binaries are upgraded. + Restart to load the new version: + + ```shell + sudo gitlab-ctl restart + ``` + +## Upgrade the Patroni node + +Patroni is used to achiece high availabilty with PostgreSQL. + +If a PostgreSQL major version upgrade is required, +[follow the major version process](../administration/postgresql/replication_and_failover.md#upgrading-postgresql-major-version-in-a-patroni-cluster). + +The upgrade process for all other versions is performed on all replicas first. +After they're upgraded, a cluster failover occurs from the leader to one of the upgraded +replicas. This ensures that only one failover is needed, and once complete the new +leader will be upgraded. + +Follow the following process: + +1. Identify the leader and replica nodes, and [verify that the cluster is healthy](../administration/postgresql/replication_and_failover.md#check-replication-status). + Run on a database node: + + ```shell + sudo gitlab-ctl patroni members + ``` + +1. Upgrade the GitLab package on one of the replica nodes: + + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee + + # Centos/RHEL + sudo yum install gitlab-ee + ``` + +1. Restart to load the new version: + + ```shell + sudo gitlab-ctl restart + ``` + +1. [Verify that the cluster is healthy](../administration/postgresql/replication_and_failover.md#check-replication-status). +1. Repeat these steps for the other replica: upgrade, restart, health check. +1. Upgrade the leader node following the same package upgrade as the replicas. +1. Restart all services on the leader node to load the new version, and also + trigger a cluster failover: + + ```shell + sudo gitlab-ctl restart + ``` + +1. [Check the cluster is healthy](../administration/postgresql/replication_and_failover.md#check-replication-status) + +## Upgrade the PgBouncer nodes + +If you run PgBouncer on your Rails (application) nodes, then +PgBouncer are upgraded as part of the application server upgrade. + +Upgrade the PgBouncer nodes: + +```shell +# Debian/Ubuntu +sudo apt-get update && sudo apt-get install gitlab-ee + +# Centos/RHEL +sudo yum install gitlab-ee +``` + +## Upgrade the Redis node + +Upgrade a standalone Redis server by updating the GitLab package: + +```shell +# Debian/Ubuntu +sudo apt-get update && sudo apt-get install gitlab-ee + +# Centos/RHEL +sudo yum install gitlab-ee +``` + +## Upgrade Redis HA (using Sentinel) **(PREMIUM SELF)** + +Follow [the zero downtime instructions](zero_downtime.md#use-redis-ha-using-sentinel) +for upgrading your Redis HA cluster. + +## Upgrade the Rails nodes (Puma / Sidekiq) + +All the Puma and Sidekiq processes were previously shut down. On each node: + +1. Ensure `/etc/gitlab/skip-auto-reconfigure` does not exist. +1. Check that Puma and Sidekiq are shut down: + + ```shell + ps -ef | egrep 'puma: | puma | sidekiq ' + ``` + +Select one node that runs Puma. This will be your deploy node, and is responsible for +running all database migrations. On the deploy node: + +1. Ensure the server is configured to permit regular migrations. Check that + `/etc/gitlab/gitlab.rb` does not contain `gitlab_rails['auto_migrate'] = false`. + Either set it specifically `gitlab_rails['auto_migrate'] = true` or omit it + for the default behavior (`true`). + +1. If you're using PgBouncer: + + You must bypass PgBouncer and connect directly to PostgreSQL + before running migrations. + + Rails uses an advisory lock when attempting to run a migration to prevent + concurrent migrations from running on the same database. These locks are + not shared across transactions, resulting in `ActiveRecord::ConcurrentMigrationError` + and other issues when running database migrations using PgBouncer in transaction + pooling mode. + + 1. If you're running Patroni, find the leader node. Run on a database node: + + ```shell + sudo gitlab-ctl patroni members + ``` + + 1. Update `gitlab.rb` on the deploy node. Change `gitlab_rails['db_host']` + and `gitlab_rails['db_port']` to either: + + - The host and port for your database server (unclustered PostgreSQL). + - The host and port for your cluster leader if you're running Patroni. + + 1. Apply the changes: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Upgrade the GitLab package: + + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee + + # Centos/RHEL + sudo yum install gitlab-ee + ``` + +1. If you modified `gitlab.rb` on the deploy node to bypass PgBouncer: + 1. Update `gitlab.rb` on the deploy node. Change `gitlab_rails['db_host']` + and `gitlab_rails['db_port']` back to your PgBouncer settings. + 1. Apply the changes: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. To ensure all services are running the upgraded version, and (if applicable) accessing + the database using PgBouncer, restart all services on the deploy node: + + ```shell + sudo gitlab-ctl restart + ``` + +Next, upgrade all the other Puma and Sidekiq nodes. The setting `gitlab_rails['auto_migrate']` can be +set to anything in `gitlab.rb` on these nodes. + +They can be upgraded in parallel: + +1. Upgrade the GitLab package: + + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee + + # Centos/RHEL + sudo yum install gitlab-ee + ``` + +1. Ensure all services are restarted: + + ```shell + sudo gitlab-ctl restart + ``` + +## Upgrade the Monitor node + +Upgrade the GitLab package: + +```shell +# Debian/Ubuntu +sudo apt-get update && sudo apt-get install gitlab-ee + +# Centos/RHEL +sudo yum install gitlab-ee +``` diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index 99457ef98d6..a3f9886ed0b 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -17,11 +17,16 @@ there are the following requirements: - You are using PostgreSQL. Starting from GitLab 12.1, MySQL is not supported. - You have set up a multi-node GitLab instance. Single-node instances do not support zero-downtime upgrades. +If you want to upgrade multiple releases or do not meet the other requirements: + +- [Upgrade a single node with downtime](package/index.md). +- [Upgrade a multi-node instance with downtime](with_downtime.md). + If you meet all the requirements above, follow these instructions in order. There are three sets of steps, depending on your deployment type: | Deployment type | Description | | --------------------------------------------------------------- | ------------------------------------------------ | -| [Gitaly Cluster](#gitaly-cluster) | GitLab CE/EE using HA architecture for Gitaly Cluster | +| [Gitaly or Gitaly Cluster](#gitaly-or-gitaly-cluster) | GitLab CE/EE using HA architecture for Gitaly or Gitaly Cluster | | [Multi-node / PostgreSQL HA](#use-postgresql-ha) | GitLab CE/EE using HA architecture for PostgreSQL | | [Multi-node / Redis HA](#use-redis-ha-using-sentinel) | GitLab CE/EE using HA architecture for Redis | | [Geo](#geo-deployment) | GitLab EE with Geo enabled | @@ -176,66 +181,84 @@ load balancer to latest GitLab version. sudo gitlab-rake db:migrate ``` -### Gitaly Cluster +### Gitaly or Gitaly Cluster -[Gitaly Cluster](../administration/gitaly/praefect.md) is built using -Gitaly and the Praefect component. It has its own PostgreSQL database, independent of the rest of -the application. +Gitaly nodes can be located on their own server, either as part of a sharded setup, or as part of +[Gitaly Cluster](../administration/gitaly/praefect.md). -Before you update the main application you need to update Praefect. -Out of your Praefect nodes, pick one to be your Praefect deploy node. -This is where you install the new Omnibus package first and run -database migrations. +Before you update the main GitLab application you must (in order): -**Praefect deploy node** +1. Upgrade the Gitaly nodes that reside on separate servers. +1. Upgrade Praefect if using Gitaly Cluster. -- Create an empty file at `/etc/gitlab/skip-auto-reconfigure`. This prevents upgrades from running `gitlab-ctl reconfigure`, which by default automatically stops GitLab, runs all database migrations, and restarts GitLab: +#### Upgrade Gitaly nodes - ```shell - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` +Upgrade the Gitaly nodes one at a time to ensure access to Git repositories is maintained: -- Ensure that `praefect['auto_migrate'] = true` is set in `/etc/gitlab/gitlab.rb` +```shell +# Debian/Ubuntu +sudo apt-get update && sudo apt-get install gitlab-ee -**All Praefect nodes _excluding_ the Praefect deploy node** +# Centos/RHEL +sudo yum install gitlab-ee +``` -- To prevent `reconfigure` from automatically running database migrations, ensure that `praefect['auto_migrate'] = false` is set in `/etc/gitlab/gitlab.rb`. +If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. -**Praefect deploy node** +#### Upgrade Praefect -- Update the GitLab package: +From the Praefect nodes, select one to be your Praefect deploy node. You install the new Omnibus package on the deploy +node first and run database migrations. - ```shell - # Debian/Ubuntu - sudo apt-get update && sudo apt-get install gitlab-ee +1. On the **Praefect deploy node**: - # Centos/RHEL - sudo yum install gitlab-ee - ``` + 1. Create an empty file at `/etc/gitlab/skip-auto-reconfigure`. This prevents upgrades from running `gitlab-ctl reconfigure`, + which by default automatically stops GitLab, runs all database migrations, and restarts GitLab: - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. + ```shell + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` -- To apply the Praefect database migrations and restart Praefect, run: + 1. Ensure that `praefect['auto_migrate'] = true` is set in `/etc/gitlab/gitlab.rb`. - ```shell - sudo gitlab-ctl reconfigure - ``` +1. On all **remaining Praefect nodes**, ensure that `praefect['auto_migrate'] = false` is + set in `/etc/gitlab/gitlab.rb` to prevent `reconfigure` from automatically running database migrations. -**All Praefect nodes _excluding_ the Praefect deploy node** +1. On the **Praefect deploy node**: -- Update the GitLab package: + 1. Upgrade the GitLab package: - ```shell - sudo apt-get update && sudo apt-get install gitlab-ee - ``` + ```shell + # Debian/Ubuntu + sudo apt-get update && sudo apt-get install gitlab-ee - If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the above command. + # Centos/RHEL + sudo yum install gitlab-ee + ``` -- Ensure nodes are running the latest code: + If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the command above. - ```shell - sudo gitlab-ctl reconfigure - ``` + 1. To apply the Praefect database migrations and restart Praefect, run: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. On all **remaining Praefect nodes**: + + 1. Upgrade the GitLab package: + + ```shell + sudo apt-get update && sudo apt-get install gitlab-ee + ``` + + If you are a Community Edition user, replace `gitlab-ee` with `gitlab-ce` in the command above. + + 1. Ensure nodes are running the latest code: + + ```shell + sudo gitlab-ctl reconfigure + ``` ### Use PostgreSQL HA @@ -290,7 +313,7 @@ node throughout the process. - If you're using PgBouncer: - You need to bypass PgBouncer and connect directly to the database master + You need to bypass PgBouncer and connect directly to the database leader before running migrations. Rails uses an advisory lock when attempting to run a migration to prevent @@ -299,7 +322,7 @@ node throughout the process. and other issues when running database migrations using PgBouncer in transaction pooling mode. - To find the master node, run the following on a database node: + To find the leader node, run the following on a database node: ```shell sudo gitlab-ctl patroni members @@ -307,7 +330,7 @@ node throughout the process. Then, in your `gitlab.rb` file on the deploy node, update `gitlab_rails['db_host']` and `gitlab_rails['db_port']` with the database - master's host and port. + leader's host and port. - To get the regular database migrations and latest code in place, run @@ -589,7 +612,7 @@ You can only upgrade one minor release at a time. This section describes the steps required to upgrade a multi-node / HA deployment with Geo. Some steps must be performed on a particular node. This -node is known as the “deploy node” and is noted through the following +node is known as the "deploy node" and is noted through the following instructions. Updates must be performed in the following order: @@ -673,7 +696,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure 1. If you're using PgBouncer: - You need to bypass PgBouncer and connect directly to the database master + You need to bypass PgBouncer and connect directly to the database leader before running migrations. Rails uses an advisory lock when attempting to run a migration to prevent @@ -682,7 +705,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure and other issues when running database migrations using PgBouncer in transaction pooling mode. - To find the master node, run the following on a database node: + To find the leader node, run the following on a database node: ```shell sudo gitlab-ctl patroni members @@ -690,7 +713,7 @@ sudo touch /etc/gitlab/skip-auto-reconfigure Then, in your `gitlab.rb` file on the deploy node, update `gitlab_rails['db_host']` and `gitlab_rails['db_port']` with the database - master's host and port. + leader's host and port. 1. To get the regular database migrations and latest code in place, run diff --git a/doc/user/admin_area/appearance.md b/doc/user/admin_area/appearance.md index 2083cb06f93..4c3bdde223b 100644 --- a/doc/user/admin_area/appearance.md +++ b/doc/user/admin_area/appearance.md @@ -7,7 +7,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/customization/branded_login_page. # GitLab Appearance **(FREE SELF)** -There are several options for customizing the appearance of a self-managed instance +Several options are available for customizing the appearance of a self-managed instance of GitLab. To access these settings: 1. On the top bar, select **Menu > Admin**. @@ -19,11 +19,13 @@ By default, the navigation bar has the GitLab logo, but this can be customized w any image desired. It is optimized for images 28px high (any width), but any image can be used (less than 1 MB) and it is automatically resized. -After you select and upload an image, click **Update appearance settings** at the bottom +After you select and upload an image, select **Update appearance settings** at the bottom of the page to activate it in the GitLab instance. NOTE: -GitLab pipeline emails also display the custom logo. +GitLab pipeline emails also display the custom logo, unless the logo is in SVG format. If the +custom logo is in SVG format, the default logo is used instead because the SVG format is not +supported by many email clients. ## Favicon @@ -31,7 +33,7 @@ By default, the favicon (used by the browser as the tab icon, as well as the CI uses the GitLab logo, but this can be customized with any icon desired. It must be a 32x32 `.png` or `.ico` image. -After you select and upload an icon, click **Update appearance settings** at the bottom +After you select and upload an icon, select **Update appearance settings** at the bottom of the page to activate it in the GitLab instance. ## System header and footer messages @@ -48,7 +50,7 @@ as the header and footer messages can only be a single line. If desired, you can select **Enable header and footer in emails** to have the text of the header and footer added to all emails sent by the GitLab instance. -After you add a message, click **Update appearance settings** at the bottom of the page +After you add a message, select **Update appearance settings** at the bottom of the page to activate it in the GitLab instance. ## Sign in / Sign up pages @@ -60,8 +62,8 @@ The optimal size for the logo is 640x360px, but any image can be used (below 1MB and it is resized automatically. The logo image appears between the title and the description, on the left of the sign-up page. -After you add a message, click **Update appearance settings** at the bottom of the page -to activate it in the GitLab instance. You can also click on the **Sign-in page** button, +After you add a message, select **Update appearance settings** at the bottom of the page +to activate it in the GitLab instance. You can also select **Sign-in page**, to review the saved appearance settings: NOTE: @@ -75,9 +77,9 @@ You can make full use of [Markdown](../markdown.md) in the description: The message is displayed below the **New Project** message, on the left side of the **New project page**. -After you add a message, click **Update appearance settings** at the bottom of the page -to activate it in the GitLab instance. You can also click on the **New project page** -button, which brings you to the new project page so you can review the change. +After you add a message, select **Update appearance settings** at the bottom of the page +to activate it in the GitLab instance. You can also select **New project page**, +which brings you to the new project page so you can review the change. ## Libravatar diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md index 9d6dcf30908..9d4c1ffe375 100644 --- a/doc/user/admin_area/broadcast_messages.md +++ b/doc/user/admin_area/broadcast_messages.md @@ -70,7 +70,7 @@ To add a broadcast message: 1. Select the **Dismissable** checkbox to enable users to dismiss the broadcast message. 1. Optional. Select **Target roles** to only show the broadcast message to users with the selected roles. The message displays on group, subgroup, and project pages, and does not display in Git remote responses. 1. If required, add a **Target Path** to only show the broadcast message on URLs matching that path. You can use the wildcard character `*` to match multiple URLs, for example `mygroup/myproject*`. -1. Select a date for the message to start and end. +1. Select a date and time (UTC) for the message to start and end. 1. Select **Add broadcast message**. When a broadcast message expires, it no longer displays in the user interface but is still listed in the @@ -78,7 +78,7 @@ list of broadcast messages. ## Edit a broadcast message -If you need to make changes to a broadcast message, you can edit it. +If you must make changes to a broadcast message, you can edit it. To edit a broadcast message: diff --git a/doc/user/admin_area/custom_project_templates.md b/doc/user/admin_area/custom_project_templates.md index 976d8e4efcf..c3c580a91c3 100644 --- a/doc/user/admin_area/custom_project_templates.md +++ b/doc/user/admin_area/custom_project_templates.md @@ -17,7 +17,7 @@ Every project in the group, but not its subgroups, can be selected when a new pr is created, based on the user's access permissions: - Public projects can be selected by any signed-in user as a template for a new project, - if all enabled [project features](../project/settings/index.md#sharing-and-permissions) + if all enabled [project features](../project/settings/index.md#configure-project-visibility-features-and-permissions) except for **GitLab Pages** and **Security & Compliance** are set to **Everyone With Access**. The same applies to internal projects. - Private projects can be selected only by users who are members of the projects. diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index 43dce1921f4..3c33578b88f 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 262bb2cc931..8f36021084e 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -105,7 +105,7 @@ To filter only projects in that namespace, select from the **Namespace** dropdow You can combine the filter options. For example, to list only public projects with `score` in their name: -1. Click the **Public** tab. +1. Select the **Public** tab. 1. Enter `score` in the **Filter by name...** input box. ### Administering Users @@ -115,7 +115,7 @@ You can administer all users in the GitLab instance from the Admin Area's Users 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Users**. -To list users matching a specific criteria, click on one of the following tabs on the **Users** page: +To list users matching a specific criteria, select one of the following tabs on the **Users** page: - **Active** - **Admins** @@ -135,13 +135,13 @@ For each user, the following are listed: 1. Date of account creation 1. Date of last activity -To edit a user, click the **Edit** button in that user's -row. To delete the user, or delete the user and their contributions, click the cog dropdown in +To edit a user, select the **Edit** button in that user's +row. To delete the user, or delete the user and their contributions, select the cog dropdown in that user's row, and select the desired option. To change the sort order: -1. Click the sort dropdown. +1. Select the sort dropdown. 1. Select the desired order. By default the sort dropdown shows **Name**. @@ -240,17 +240,17 @@ To access the Groups page: 1. On the left sidebar, select **Overview > Groups**. For each group, the page displays their name, description, size, number of projects in the group, -number of members, and whether the group is private, internal, or public. To edit a group, click -the **Edit** button in that group's row. To delete the group, click the **Delete** button in +number of members, and whether the group is private, internal, or public. To edit a group, select +the **Edit** button in that group's row. To delete the group, select the **Delete** button in that group's row. -To change the sort order, click the sort dropdown and select the desired order. The default +To change the sort order, select the sort dropdown and select the desired order. The default sort order is by **Last created**. To search for groups by name, enter your criteria in the search field. The group search is case insensitive, and applies partial matching. -To [Create a new group](../group/index.md#create-a-group) click **New group**. +To [Create a new group](../group/index.md#create-a-group) select **New group**. ### Administering Topics @@ -286,7 +286,7 @@ To access the Jobs page: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Jobs**. All jobs are listed, in descending order of job ID. -1. Click the **All** tab to list all jobs. Click the **Pending**, **Running**, or **Finished** +1. Select the **All** tab to list all jobs. Select the **Pending**, **Running**, or **Finished** tab to list only jobs of that status. For each job, the following details are listed: @@ -313,14 +313,7 @@ To access the **Runners** page: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Runners**. -The **Runners** page features: - -- A description of runners and their possible states. -- Instructions on installing a runner. -- A list of all registered runners. - -Runners are listed in descending order by the date they were created, by default. You can change -the sort order to *Last Contacted* from the dropdown beside the search field. +#### Search and filter runners To search runners' descriptions: @@ -336,20 +329,18 @@ You can also filter runners by status, type, and tag. To filter:  +#### Runner attributes + For each runner, the following attributes are listed: | Attribute | Description | |--------------|-------------| -| Type/State | One or more of the following states: shared, group, specific, locked, or paused | -| Runner token | Partial token used to identify the runner, and which the runner uses to communicate with the GitLab instance | -| Runner ID | Numerical ID of the runner | -| Description | Description given to the runner | -| Version | GitLab Runner version | -| IP address | IP address of the host on which the runner is registered | -| Projects | Number of projects to which the runner is assigned | -| Jobs | Total of jobs run by the runner | -| Tags | Tags associated with the runner | -| Last contact | Timestamp indicating when the runner last contacted the GitLab instance | +| Status | The status of the runner. In [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/22224), for the **Ultimate** tier, the upgrade status is available. | +| Runner details | Information about the runner, including partial token and details about the computer the runner was registered from. | +| Version | GitLab Runner version. | +| Jobs | Total number of jobs run by the runner. | +| Tags | Tags associated with the runner. | +| Last contact | Timestamp indicating when the runner last contacted the GitLab instance. | You can also edit, pause, or remove each runner. diff --git a/doc/user/admin_area/license.md b/doc/user/admin_area/license.md index 773f91d3076..ac5e5ded859 100644 --- a/doc/user/admin_area/license.md +++ b/doc/user/admin_area/license.md @@ -36,7 +36,7 @@ To activate your instance with an activation code: The subscription is activated. -If you have an offline or air gapped environment, +If you have an offline environment, [activate GitLab EE with a license file or key](license_file.md) instead. If you have questions or need assistance activating your instance, @@ -64,6 +64,6 @@ This error occurs when you use an activation code to activate your instance, but You may have connectivity issues due to the following reasons: -- **You have an offline or air gapped environment**: Configure your setup to allow connection to GitLab servers. If connection to GitLab servers is not possible, contact [GitLab support](https://about.gitlab.com/support/#contact-support) to request a license key. +- **You have an offline environment**: Configure your setup to allow connection to GitLab servers. If connection to GitLab servers is not possible, contact [GitLab support](https://about.gitlab.com/support/#contact-support) to request a license key. - **Firewall settings**: Enable an encrypted HTTPS connection from your GitLab instance to `customers.gitlab.com` (with IP addresses 104.18.26.123 and 104.18.27.123) on port 443. - **Customers Portal is not operational**: To check for performance or service disruptions, check the Customers Portal [status](https://status.gitlab.com/). diff --git a/doc/user/admin_area/license_file.md b/doc/user/admin_area/license_file.md index ff9e87680f9..be1b1a16e29 100644 --- a/doc/user/admin_area/license_file.md +++ b/doc/user/admin_area/license_file.md @@ -4,6 +4,8 @@ group: Provision info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- +<!-- To promote the workflow described in license.md, this page is not included in global left nav. --> + # Activate GitLab EE with a license file or key If you receive a license file from GitLab (for example, for a trial), you can @@ -47,6 +49,17 @@ WARNING: These methods only add a license at the time of installation. To renew or upgrade a license, add the license in the **Admin Area** in the web user interface. +## Submit license usage data + +If you use a license file or key to activate your instance in an offline environment, you must submit your license +usage data monthly. +To submit the data, [export your license usage](../../subscriptions/self_managed/index.md#export-your-license-usage) +and send it by email to the renewals service, `renewals-service@customers.gitlab.com`. + +If you don't submit your data each month after your subscription start date, a banner displays to remind you to +submit your data. The banner displays in the **Admin Area** on the **Dashboard** and on the **Subscription** +pages. You can only dismiss it until the following month after you submit your license usage data. + ## What happens when your license expires Fifteen days before the license expires, a notification banner with the upcoming expiration diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index ed7fdfe2111..526e8cd17da 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -22,7 +22,7 @@ To enable merge request approval settings for an instance: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **{push-rules}** **Push Rules**, and expand **Merge request approvals**. 1. Choose the required options. -1. Click **Save changes**. +1. Select **Save changes**. ## Available rules @@ -38,4 +38,4 @@ Merge request approval settings that can be set at an instance level are: See also the following, which are affected by instance-level rules: - [Project merge request approval rules](../project/merge_requests/approvals/index.md). -- [Group merge request approval settings](../group/index.md#group-approval-settings) available in GitLab 13.9 and later. +- [Group merge request approval settings](../group/index.md#group-merge-request-approval-settings) available in GitLab 13.9 and later. diff --git a/doc/user/admin_area/moderate_users.md b/doc/user/admin_area/moderate_users.md index 53c08d8cbc1..dc6ff96c31f 100644 --- a/doc/user/admin_area/moderate_users.md +++ b/doc/user/admin_area/moderate_users.md @@ -171,8 +171,12 @@ Users can also be deactivated using the [GitLab API](../../api/users.md#deactiva > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/320875) in GitLab 14.0. -Administrators can enable automatic deactivation of users who have not signed in, or have no activity -in the last 90 days. To do this: +Administrators can enable automatic deactivation of users who either: + +- Were created more than a week ago and have not signed in. +- Have no activity in the last 90 days. + +To do this: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. @@ -246,3 +250,33 @@ A banned user can be unbanned using the Admin Area. To do this: The user's state is set to active and they consume a [seat](../../subscriptions/self_managed/index.md#billable-users). + +### Delete a user + +Use the Admin Area to delete users. + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Overview > Users**. +1. Select the **Banned** tab. +1. Optional. Select a user. +1. Select the **{settings}** **User administration** dropdown list. +1. Select **Delete user**. +1. Type the username. +1. Select **Delete user**. + +NOTE: +You can only delete a user if there are inherited or direct owners of a group. You cannot delete a user if they are the only group owner. + +You can also delete a user and their contributions, such as merge requests, issues, and groups of which they are the only group owner. + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Overview > Users**. +1. Select the **Banned** tab. +1. Optional. Select a user. +1. Select the **{settings}** **User administration** dropdown list. +1. Select **Delete user and contributions**. +1. Type the username. +1. Select **Delete user and contributions**. + +NOTE: +Before 15.1, additionally groups of which deleted user were the only owner among direct members were deleted. diff --git a/doc/user/admin_area/monitoring/background_migrations.md b/doc/user/admin_area/monitoring/background_migrations.md index b666c0c5ad2..53d5056bb65 100644 --- a/doc/user/admin_area/monitoring/background_migrations.md +++ b/doc/user/admin_area/monitoring/background_migrations.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index 213bddec325..84c7fa3c419 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -12,14 +12,14 @@ database connection, Redis connection, and access to the file system. These endpoints [can be provided to schedulers like Kubernetes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) to hold traffic until the system is ready or restart the container as needed. -## IP whitelist +## IP allowlist -To access monitoring resources, the requesting client IP needs to be included in a whitelist. -For details, see [how to add IPs to a whitelist for the monitoring endpoints](../../../administration/monitoring/ip_whitelist.md). +To access monitoring resources, the requesting client IP needs to be included in the allowlist. +For details, see [how to add IPs to the allowlist for the monitoring endpoints](../../../administration/monitoring/ip_whitelist.md). ## Using the endpoints locally -With default whitelist settings, the probes can be accessed from localhost using the following URLs: +With default allowlist settings, the probes can be accessed from localhost using the following URLs: ```plaintext GET http://localhost/-/health diff --git a/doc/user/admin_area/reporting/spamcheck.md b/doc/user/admin_area/reporting/spamcheck.md index b1ec203cffc..2360c1f2899 100644 --- a/doc/user/admin_area/reporting/spamcheck.md +++ b/doc/user/admin_area/reporting/spamcheck.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 9905298784a..bedd648b3e7 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -112,7 +112,7 @@ To change the default global prefix: 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. 1. Fill in the **Personal Access Token prefix** field. -1. Click **Save changes**. +1. Select **Save changes**. You can also configure the prefix by using the [settings API](../../../api/settings.md). @@ -148,17 +148,17 @@ These settings can be found in: - Each project's settings: 1. From the Project's homepage, navigate to **Settings > General**. 1. Fill in the **Repository size limit (MB)** field in the **Naming, topics, avatar** section. - 1. Click **Save changes**. + 1. Select **Save changes**. - Each group's settings: 1. From the Group's homepage, navigate to **Settings > General**. 1. Fill in the **Repository size limit (MB)** field in the **Naming, visibility** section. - 1. Click **Save changes**. + 1. Select **Save changes**. - GitLab global settings: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. 1. Fill in the **Size limit per repository (MB)** field. - 1. Click **Save changes**. + 1. Select **Save changes**. The first push of a new project, including LFS objects, is checked for size. If the sum of their sizes exceeds the maximum allowed repository size, the push @@ -186,13 +186,13 @@ To set a limit on how long these sessions are valid: 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. 1. Fill in the **Session duration for Git operations when 2FA is enabled (minutes)** field. -1. Click **Save changes**. +1. Select **Save changes**. ## Limit the lifetime of SSH keys **(ULTIMATE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.6 [with a flag](../../../administration/feature_flags.md) named `ff_limit_ssh_key_lifetime`. Disabled by default. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/346753) in GitLab 14.6. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.7. [Feature flag ff_limit_ssh_key_lifetime](https://gitlab.com/gitlab-org/gitlab/-/issues/347408) removed. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/1007) in GitLab 14.7. [Feature flag `ff_limit_ssh_key_lifetime`](https://gitlab.com/gitlab-org/gitlab/-/issues/347408) removed. Users can optionally specify a lifetime for [SSH keys](../../ssh.md). @@ -213,7 +213,7 @@ To set a lifetime on how long SSH keys are valid: 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. 1. Fill in the **Maximum allowable lifetime for SSH keys (days)** field. -1. Click **Save changes**. +1. Select **Save changes**. Once a lifetime for SSH keys is set, GitLab: @@ -261,7 +261,7 @@ To set a lifetime on how long access tokens are valid: 1. On the left sidebar, select **Settings > General**. 1. Expand the **Account and limit** section. 1. Fill in the **Maximum allowable lifetime for access tokens (days)** field. -1. Click **Save changes**. +1. Select **Save changes**. Once a lifetime for access tokens is set, GitLab: @@ -328,4 +328,4 @@ Your push has been rejected, because this repository has exceeded its size limit To resolve this problem, either of these options helps in the short- to middle-term: - Increase the [repository size limit](#repository-size-limit). -- [Reduce the repo size](../../project/repository/reducing_the_repo_size_using_git.md). +- [Reduce the repository size](../../project/repository/reducing_the_repo_size_using_git.md). diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index 170d3cf4c90..7f37c99259a 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -39,6 +39,23 @@ You can set all new projects to have the instance's shared runners available by Any time a new project is created, the shared runners are available. +## Shared runners CI/CD minutes + +As an administrator you can set either a global or namespace-specific +limit on the number of [CI/CD minutes](../../../ci/pipelines/cicd_minutes.md) you can use. + +## Enable a specific runner for multiple projects + +To enable a specific runner for one or more projects: + +1. On the top bar, select **Menu > Admin**. +1. From the left sidebar, select **Overview > Runners**. +1. Select the runner you want to edit. +1. In the top right, select **Edit** (**{pencil}**). +1. Under **Restrict projects for this runner**, search for a project. +1. To the left of the project, select **Enable**. +1. Repeat this process for each additional project. + ## Add a message for shared runners To display details about the instance's shared runners in all projects' @@ -143,10 +160,6 @@ A new pipeline must run before the latest artifacts can expire and be deleted. NOTE: All application settings have a [customizable cache expiry interval](../../../administration/application_settings_cache.md) which can delay the settings affect. -## Shared runners CI/CD minutes - -As an administrator you can set either a global or namespace-specific limit on the number of [CI/CD minutes](../../../ci/pipelines/cicd_minutes.md) you can use. - ## Archive jobs Archiving jobs is useful for reducing the CI/CD footprint on the system by removing some diff --git a/doc/user/admin_area/settings/floc.md b/doc/user/admin_area/settings/floc.md index 6bfc6dfbebd..dccab461b85 100644 --- a/doc/user/admin_area/settings/floc.md +++ b/doc/user/admin_area/settings/floc.md @@ -26,7 +26,7 @@ To enable it: 1. On the left sidebar, select **Settings > General**. 1. Expand **Federated Learning of Cohorts**. 1. Check the box. -1. Click **Save changes**. +1. Select **Save changes**. <!-- ## Troubleshooting diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index 970897fd8da..034a432c570 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -79,7 +79,7 @@ The **Geo** setting contains: The **Integrations** settings contain: -- [Elasticsearch](../../../integration/elasticsearch.md#enable-advanced-search) - +- [Elasticsearch](../../../integration/advanced_search/elasticsearch.md#enable-advanced-search) - Elasticsearch integration. Elasticsearch AWS IAM. - [Kroki](../../../administration/integration/kroki.md#enable-kroki-in-gitlab) - Allow rendering of diagrams in AsciiDoc and Markdown documents using [kroki.io](https://kroki.io). @@ -160,7 +160,7 @@ The **Preferences** settings contain: The **Reporting** settings contain: - [Spam and Anti-bot Protection](../../../integration/recaptcha.md) - - Enable anti-spam services, like reCAPTCHA, Akismet or [Spamcheck](../reporting/spamcheck.md), and set IP limits. + Enable anti-spam services, like reCAPTCHA, Akismet, or [Spamcheck](../reporting/spamcheck.md), and set IP limits. - [Abuse reports](../review_abuse_reports.md) - Set notification email for abuse reports. ### Repository diff --git a/doc/user/admin_area/settings/project_integration_management.md b/doc/user/admin_area/settings/project_integration_management.md index aadabe4d6ad..010ba6a12fc 100644 --- a/doc/user/admin_area/settings/project_integration_management.md +++ b/doc/user/admin_area/settings/project_integration_management.md @@ -25,7 +25,7 @@ Only the complete settings for an integration can be inherited. Per-field inheri 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Integrations**. 1. Select an integration. -1. Enter configuration details and click **Save changes**. +1. Enter configuration details and select **Save changes**. WARNING: This may affect all or most of the groups and projects on your GitLab instance. Please review the details @@ -57,7 +57,7 @@ integration on all non-configured groups and projects by default. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > Integrations**. 1. Select an integration. -1. Click **Reset** and confirm. +1. Select **Reset** and confirm. Resetting an instance-level default setting removes the integration from all projects that have the integration set to use default settings. @@ -79,7 +79,7 @@ for an integration. 1. Navigate to the group's **Settings > Integrations**. 1. Select an integration. -1. Enter configuration details and click **Save changes**. +1. Enter configuration details and select **Save changes**. WARNING: This may affect all or most of the subgroups and projects belonging to the group. Please review the details below. @@ -112,7 +112,7 @@ integration on all non-configured groups and projects by default. 1. Navigate to the group's **Settings > Integrations**. 1. Select an integration. -1. Click **Reset** and confirm. +1. Select **Reset** and confirm. Resetting a group-level default setting removes integrations that use default settings and belong to a project or subgroup of the group. @@ -124,7 +124,7 @@ Resetting a group-level default setting removes integrations that use default se 1. Choose the integration you want to enable or update. 1. From the drop-down, select **Use default settings**. 1. Ensure the toggle is set to **Enable integration**. -1. Click **Save changes**. +1. Select **Save changes**. ## Use custom settings for a group or project integration @@ -132,4 +132,4 @@ Resetting a group-level default setting removes integrations that use default se 1. Choose the integration you want to enable or update. 1. From the drop-down, select **Use custom settings**. 1. Ensure the toggle is set to **Enable integration** and enter all required settings. -1. Click **Save changes**. +1. Select **Save changes**. diff --git a/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md index 2819a18d361..fce6179f5cf 100644 --- a/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md +++ b/doc/user/admin_area/settings/rate_limit_on_pipelines_creation.md @@ -31,3 +31,4 @@ To limit the number of pipeline requests: 1. Expand **Pipelines Rate Limits**. 1. Under **Max requests per minute**, enter a value greater than `0`. 1. Select **Save changes**. +1. Enable `ci_enforce_throttle_pipelines_creation` feature flag to enable the rate limit. diff --git a/doc/user/admin_area/settings/rate_limit_on_users_api.md b/doc/user/admin_area/settings/rate_limit_on_users_api.md index 7954055f38b..5eed989f73f 100644 --- a/doc/user/admin_area/settings/rate_limit_on_users_api.md +++ b/doc/user/admin_area/settings/rate_limit_on_users_api.md @@ -1,7 +1,7 @@ --- type: reference stage: Manage -group: Authentication & Authorization +group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/user/admin_area/settings/terms.md b/doc/user/admin_area/settings/terms.md index 693b3e6c7b6..870fa7ad18e 100644 --- a/doc/user/admin_area/settings/terms.md +++ b/doc/user/admin_area/settings/terms.md @@ -23,7 +23,7 @@ To enforce acceptance of a Terms of Service and Privacy Policy: 1. Check the **All users must accept the Terms of Service and Privacy Policy to access GitLab** checkbox. 1. Input the text of the **Terms of Service and Privacy Policy**. You can use [Markdown](../../markdown.md) in this text box. -1. Click **Save changes**. +1. Select **Save changes**. For each update to the terms, a new version is stored. When a user accepts or declines the terms, GitLab records which version they accepted or declined. diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index ce949999fb8..c74906c2762 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -48,7 +48,7 @@ tier. Users can continue to access the features in a paid tier without sharing u ### Features available in 14.4 and later - [Repository size limit](../settings/account_and_limit_settings.md#repository-size-limit). -- [Restrict group access by IP address](../../group/index.md#restrict-group-access-by-ip-address). +- [Group access restriction by IP address](../../group/index.md#group-access-restriction-by-ip-address). NOTE: Registration is not yet required for participation, but may be added in a future milestone. diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index 3eae0d0ff90..8a9db68b34f 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -33,49 +33,70 @@ on the instance. To alter which roles have permission to create projects: - Developers and Maintainers. 1. Select **Save changes**. -## Restrict project deletion to Administrators **(PREMIUM SELF)** +## Restrict project deletion to administrators **(PREMIUM SELF)** -Anyone with the **Owner** role, either at the project or group level, can -delete a project. To allow only users with administrator access to delete projects: +> User interface [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -1. Sign in to GitLab as a user with Administrator access level. +By default both administrators and anyone with the **Owner** role can delete a project. To restrict project deletion to only administrators: + +1. Sign in to GitLab as a user with administrator access. 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Settings > General**. 1. Expand the **Visibility and access controls** section. -1. Scroll to **Default project deletion protection**, and select **Only admins can delete project**. +1. Scroll to: + - (GitLab 15.1 and later) **Allowed to delete projects**, and select **Administrators**. + - (GitLab 15.0 and earlier) **Default project deletion projection** and select **Only admins can delete project**. 1. Select **Save changes**. -## Default delayed project deletion **(PREMIUM SELF)** +## Deletion protection **(PREMIUM SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2 for groups created after August 12, 2021. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2 for groups created after August 12, 2021. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) from default delayed project deletion in GitLab 15.1. +> - [Enabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89466) in GitLab 15.1. -[Delayed project deletion](../../project/settings/index.md#delayed-project-deletion) allows projects in a group (not a personal namespace) -to be deleted after a period of delay. +Instance-level protection against accidental deletion of groups and projects. -To enable delayed project deletion by default in new groups: +### Retention period -1. Check the **Default delayed project deletion** checkbox. -1. Select **Save changes**. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -## Default deletion delay **(PREMIUM SELF)** +Groups and projects will remain restorable within a defined retention period. By default this is 7 days but it can be changed. +Setting the retention period to `0` means that groups and project are removed immediately and cannot be restored. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. +In GitLab 15.1 and later, the retention period must be between `1` and `90`. If the retention period was `0` before the 15.1 update, +then it will get automatically changed to `1` while also disabling deletion protection the next time any application setting is changed. -By default, a project marked for deletion is permanently removed with immediate effect. -See [delayed project deletion](../../project/settings/index.md#delayed-project-deletion) to learn more. -By default, a group marked for deletion is permanently removed after seven days. +### Delayed project deletion -WARNING: -The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to -[Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. +> User interface [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. + +Administrators can enable [delayed project deletion](../../project/settings/index.md#delayed-project-deletion) by default for +newly-created groups. Group owners can choose to disable this and existing groups will retain their existing setting. When enabled +deleted groups will remain restorable within a retention period. + +To configure delayed project deletion: + +1. Sign in to GitLab as a user with administrator access. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility and access controls** section. +1. Scroll to: + - (GitLab 15.1 and later) **Deletion projection** and select keep deleted groups and projects, and select a retention period. + - (GitLab 15.0 and earlier) **Default delayed project projection** and select **Enable delayed project deletion by + default for newly-created groups.** Then set a retention period in **Default deletion delay**. +1. Select **Save changes**. + +Deletion protection is not available for projects only (without being also being enabled for groups). + +In GitLab 15.1, and later this setting is enforced on groups when disabled and it cannot be overridden. + +### Delayed group deletion -The default period is seven days, and can be changed. Setting this period to `0` enables immediate removal -of projects or groups. +> User interface [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -To change this period: +Groups will remain restorable if the retention period is `1` or more days. -1. Select the desired option. -1. Click **Save changes**. +In GitLab 15.1 and later, delayed group deletion can be enabled by setting **Deletion projection** to **Keep deleted**. ### Override defaults and delete immediately @@ -95,7 +116,7 @@ To set the default [visibility levels for new projects](../../public_access.md): 1. Expand the **Visibility and access controls** section. 1. Select the desired default project visibility: - **Private** - Project access must be granted explicitly to each user. If this - project is part of a group, access will be granted to members of the group. + project is part of a group, access is granted to members of the group. - **Internal** - The project can be accessed by any logged in user except external users. - **Public** - The project can be accessed without any authentication. 1. Select **Save changes**. @@ -142,7 +163,9 @@ To restrict visibility levels for projects, snippets, and selected pages: 1. In the **Restricted visibility levels** section, select the desired visibility levels to restrict. If you restrict the **Public** level: - User profiles are only visible to logged in users via the Web interface. - - User attributes are only visible to authenticated users via the GraphQL API. + - User attributes via the GraphQL API are: + - Not visible in [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88020). + - Only visible to authenticated users between [GitLab 13.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33195) and GitLab 15.0. 1. Select **Save changes**. For more details on project visibility, see @@ -226,7 +249,7 @@ For example, if: To specify a custom Git clone URL for HTTP(S): 1. Enter a root URL for **Custom Git clone URL for HTTP(S)**. -1. Click on **Save changes**. +1. Select **Save changes**. NOTE: SSH clone URLs can be customized in `gitlab.rb` by setting `gitlab_rails['gitlab_ssh_host']` and @@ -239,7 +262,7 @@ These options specify the permitted types and lengths for SSH keys. To specify a restriction for each key type: 1. Select the desired option from the dropdown. -1. Click **Save changes**. +1. Select **Save changes**. For more details, see [SSH key restrictions](../../../security/ssh_keys_restrictions.md). @@ -251,6 +274,33 @@ work in every repository. They can only be re-enabled by an administrator user o  +## Configure globally-allowed IP address ranges + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87579) in GitLab 15.1 [with a flag](../../../administration/feature_flags.md) named `group_ip_restrictions_allow_global`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available +per group, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) +named `group_ip_restrictions_allow_global`. +On GitLab.com, this feature is available. + +This setting allows you to set IP address ranges to be combined with group-level IP allowlists. +It helps administrators prevent aspects of the GitLab installation from being blocked +from working as intended when an IP allowlist is used. + +For example, if the GitLab Pages daemon runs on the `10.0.0.0/24` range, specify that range in this +field, as otherwise any group-level restrictions that don't include that range cause the Pages +daemon to be unable to fetch artifacts from the pipeline runs. + +To add a IP address range to the group-level allowlist: + +1. Sign in to GitLab as a user with Administrator access level. +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility and access controls** section. +1. In **Globally-allowed IP ranges**, provide a value. +1. Select **Save changes**. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/analytics/ci_cd_analytics.md b/doc/user/analytics/ci_cd_analytics.md index f0de1e58891..920b651c094 100644 --- a/doc/user/analytics/ci_cd_analytics.md +++ b/doc/user/analytics/ci_cd_analytics.md @@ -6,13 +6,20 @@ info: To determine the technical writer assigned to the Stage/Group associated w # CI/CD analytics **(FREE)** -## Pipeline success and duration charts +Use the CI/CD analytics page to view pipeline success rates and duration, and the history of [DORA metrics](index.md#devops-research-and-assessment-dora-key-metrics) over time. -> [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/38318) to CI/CD Analytics in GitLab 12.8. +## Pipeline success and duration charts CI/CD analytics shows the history of your pipeline successes and failures, as well as how long each pipeline ran. +Pipeline statistics are gathered by collecting all available pipelines for the +project, regardless of status. The data available for each individual day is based +on when the pipeline was created. + +The total pipeline calculation includes child +pipelines and pipelines that failed with an invalid YAML. To filter pipelines based on other attributes, use the [Pipelines API](../../api/pipelines.md#list-project-pipelines). + View successful pipelines:  @@ -21,12 +28,6 @@ View pipeline duration history:  -Pipeline statistics are gathered by collecting all available pipelines for the -project regardless of status. The data available for each individual day is based -on when the pipeline was created. The total pipeline calculation includes child -pipelines and pipelines that failed with invalid YAML. If you are interested in -filtering pipelines based on other attributes, consider using the [Pipelines API](../../api/pipelines.md#list-project-pipelines). - ## View CI/CD analytics To view CI/CD analytics: @@ -43,6 +44,8 @@ frequency to the `production` environment. The environment must be part of the [production deployment tier](../../ci/environments/index.md#deployment-tier-of-environments) for its deployment information to appear on the graphs. + Deployment frequency is one of the four [DORA metrics](index.md#devops-research-and-assessment-dora-key-metrics) that DevOps teams use for measuring excellence in software delivery. + The deployment frequency chart is available for groups and projects. To view the deployment frequency chart: @@ -65,6 +68,8 @@ merge requests to be deployed to a production environment. This chart is availab - For time periods in which no merge requests were deployed, the charts render a red, dashed line. + lead time for changes is one of the four [DORA metrics](index.md#devops-research-and-assessment-dora-key-metrics) that DevOps teams use for measuring excellence in software delivery. + To view the lead time for changes chart: 1. On the top bar, select **Menu > Projects** and find your project. @@ -72,3 +77,19 @@ To view the lead time for changes chart: 1. Select the **Lead time** tab.  + +## View time to restore service chart **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356959) in GitLab 15.1 + +The time to restore service chart shows information about the median time an incident was open in a production environment. This chart is available for groups and projects. + +Time to restore service is one of the four [DORA metrics](index.md#devops-research-and-assessment-dora-key-metrics) that DevOps teams use for measuring excellence in software delivery. + +To view the time to restore service chart: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Analytics > CI/CD Analytics**. +1. Select the **Time to restore service** tab. + + diff --git a/doc/user/analytics/img/time_to_restore_service_charts_v15_1.png b/doc/user/analytics/img/time_to_restore_service_charts_v15_1.png Binary files differnew file mode 100644 index 00000000000..25aac385750 --- /dev/null +++ b/doc/user/analytics/img/time_to_restore_service_charts_v15_1.png diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 017a0c46570..91d9bd918b6 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -81,7 +81,7 @@ Deployment frequency displays in several charts: Lead time for changes measures the time to deliver a feature once it has been developed, as described in [Measuring DevOps Performance](https://devops.com/measuring-devops-performance/). -Lead time for changes displays in several charts: +Lead time for changes displays in several charts: - [Group-level value stream analytics](../group/value_stream_analytics/index.md) - [Project-level value stream analytics](value_stream_analytics.md) @@ -89,7 +89,7 @@ Lead time for changes displays in several charts: ### Time to restore service -Time to restore service measures how long it takes an organization to recover from a failure in production. +Time to restore service measures how long it takes an organization to recover from a failure in production. GitLab measures this as the average time required to close the incidents in the given time period. This assumes: @@ -97,9 +97,15 @@ in the given time period. This assumes: - Incidents and deployments have a strictly one-to-one relationship. An incident is related to only one production deployment, and any production deployment is related to no more than one incident). +Time to restore service displays in several charts: + +- [Group-level value stream analytics](../group/value_stream_analytics/index.md) +- [Project-level value stream analytics](value_stream_analytics.md) +- [CI/CD analytics](ci_cd_analytics.md) + To retrieve metrics for time to restore service, use the [GraphQL](../../api/graphql/reference/index.md) or the [REST](../../api/dora/metrics.md) APIs. -### Change failure rate +### Change failure rate Change failure rate measures the percentage of deployments that cause a failure in production. GitLab measures this as the number of incidents divided by the number of deployments to a diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 039d33a1ad8..c0f97369740 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -43,9 +43,8 @@ To view value stream analytics for your project: - In the **From** field, select a start date. - In the **To** field, select an end date. 1. Optional. Sort results by ascending or descending: - - To sort by most recent or oldest workflow item, select the **Merge requests** or **Issues** - header. The header name differs based on the stage you select. - - To sort by most or least amount of time spent in each stage, select the **Time** header. + - To sort by most recent or oldest workflow item, select the **Last event** header. + - To sort by most or least amount of time spent in each stage, select the **Duration** header. The table shows a list of related workflow items for the selected stage. Based on the stage you choose, this can be: @@ -119,7 +118,7 @@ The **Lead Time for Changes** metrics display below the **Filter results** text To view deployment metrics, you must have a [production environment configured](../../ci/environments/index.md#deployment-tier-of-environments). -Value stream analytics shows the following deployment metrics for your project: +Value stream analytics shows the following deployment metrics for your project: - Deploys: The number of successful deployments in the date range. - Deployment Frequency: The average number of successful deployments per day in the date range. diff --git a/doc/user/application_security/api_fuzzing/create_har_files.md b/doc/user/application_security/api_fuzzing/create_har_files.md index 1ba19359fde..bc987c56fe8 100644 --- a/doc/user/application_security/api_fuzzing/create_har_files.md +++ b/doc/user/application_security/api_fuzzing/create_har_files.md @@ -151,7 +151,7 @@ export it as a HAR file. 1. Select **Preserve log**. 1. Browse pages that call the API. 1. Select one or more requests. -1. Right click and select **Save all as HAR with content**. +1. Right-click and select **Save all as HAR with content**. 1. Enter a filename and select **Save**. 1. To append additional requests, select and save them to the same file. @@ -170,7 +170,7 @@ and export it as a HAR file. `Perform a request or Reload the page to see detailed information about network activity`, select **Reload** to start recording requests. 1. Select one or more requests. -1. Right click and select **Save All As HAR**. +1. Right-click and select **Save All As HAR**. 1. Enter a filename and select **Save**. 1. To append additional requests, select and save them to the same file. diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index f97e09f33bb..cbe20ecde30 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -65,7 +65,7 @@ Requirements: - Postman Collection v2.0 or v2.1 WARNING: - **NEVER** run fuzz testing against a production server. Not only can it perform *any* function that + **Never** run fuzz testing against a production server. Not only can it perform *any* function that the API can, it may also trigger bugs in the API. This includes actions like modifying and deleting data. Only run fuzzing against a test server. @@ -586,7 +586,7 @@ profile increases as the number of tests increases. | CI/CD variable | Description | |-------------------------------------------------------------|-------------| | `SECURE_ANALYZERS_PREFIX` | Specify the Docker registry base address from which to download the analyzer. | -| `FUZZAPI_VERSION` | Specify API Fuzzing container version. Defaults to `1`. | +| `FUZZAPI_VERSION` | Specify API Fuzzing container version. Defaults to `2`. | | `FUZZAPI_IMAGE_SUFFIX` | Specify a container image suffix. Defaults to none. | | `FUZZAPI_TARGET_URL` | Base URL of API testing target. | | `FUZZAPI_CONFIG` | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/276395) in GitLab 13.12, replaced with default `.gitlab/gitlab-api-fuzzing-config.yml`. API Fuzzing configuration file. | @@ -883,7 +883,7 @@ Adding some basic logging to your overrides script is useful in case the script Following our example, we provided `renew_token.py` in the environmental variable `FUZZAPI_OVERRIDES_CMD`. Please notice two things in the script: - Log file is saved in the location indicated by the environment variable `CI_PROJECT_DIR`. -- Log file name should match `gl-*.log`. +- Log filename should match `gl-*.log`. ```python #!/usr/bin/env python @@ -1296,7 +1296,7 @@ variables: The `api-fuzzing-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). -### Exclude URLS +### Exclude URLs > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357195) in GitLab 14.10. @@ -1476,9 +1476,9 @@ Follow these steps to view details of a fuzzing fault: - In a project, go to the project's **{shield}** **Security & Compliance > Vulnerability Report** page. This page shows all vulnerabilities from the default branch only. - - In a merge request, go the merge request's **Security** section and click the **Expand** + - In a merge request, go the merge request's **Security** section and select the **Expand** button. API Fuzzing faults are available in a section labeled - **API Fuzzing detected N potential vulnerabilities**. Click the title to display the fault + **API Fuzzing detected N potential vulnerabilities**. Select the title to display the fault details. 1. Select the fault's title to display the fault's details. The table below describes these details. @@ -1652,11 +1652,11 @@ Steps: The Docker image for API Fuzzing must be pulled (downloaded) from the public registry and then pushed (imported) into a local registry. The GitLab container registry can be used to locally host the Docker image. This process can be performed using a special template. See [loading Docker images onto your offline host](../offline_deployments/index.md#loading-docker-images-onto-your-offline-host) for instructions. -Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-fuzzing:1` results in a valid image location. +Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-security:2` results in a valid image location. -For example, the below line sets a registry for the image `registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:1`: +For example, the below line sets a registry for the image `registry.gitlab.com/security-products/api-security:2`: -`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"` +`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products"` NOTE: Setting `SECURE_ANALYZERS_PREFIX` changes the Docker image registry location for all GitLab Secure templates. diff --git a/doc/user/application_security/cluster_image_scanning/index.md b/doc/user/application_security/cluster_image_scanning/index.md index aba28a5ca89..e7a59d70b25 100644 --- a/doc/user/application_security/cluster_image_scanning/index.md +++ b/doc/user/application_security/cluster_image_scanning/index.md @@ -1,321 +1,11 @@ --- -type: reference, howto -stage: Protect -group: Container Security -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../clusters/agent/vulnerabilities.md' +remove_date: '2022-08-19' --- -# Cluster Image Scanning **(ULTIMATE)** +This document was moved to [another location](../../clusters/agent/vulnerabilities.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/) in GitLab 14.1. - -WARNING: -This analyzer is in [Alpha](../../../policy/alpha-beta-support.md#alpha-features) -and is unstable. The JSON report and CI/CD configuration may be subject to change or breakage -across GitLab releases. - -Your Kubernetes cluster may run workloads based on images that the Container Security analyzer -didn't scan. These images may therefore contain known vulnerabilities. By including an extra job in -your pipeline that scans for those security risks and displays them in the vulnerability report, you -can use GitLab to audit your Kubernetes workloads and environments. - -GitLab provides integration with open-source tools for vulnerability analysis in Kubernetes clusters: - -- [Starboard](https://github.com/aquasecurity/starboard) - -To integrate GitLab with security scanners other than those listed here, see -[Security scanner integration](../../../development/integrations/secure.md). - -You can use cluster image scanning through the following methods: - -<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> -- [The cluster image scanning analyzer](#use-the-cluster-image-scanning-analyzer-removed) ([Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in GitLab 15.0. Use [the GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) instead.) -<!--- end_remove --> -- [The GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) - -<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> - -## Use the cluster image scanning analyzer (removed) - -This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in GitLab 15.0. -Use [the GitLab agent](#cluster-image-scanning-with-the-gitlab-agent) instead. - -You can use the cluster image scanning analyzer to run cluster image scanning with [GitLab CI/CD](../../../ci/index.md). -To enable the cluster image scanning analyzer, [include the CI job](#configuration) -in your existing `.gitlab-ci.yml` file. - -### Prerequisites - -To enable cluster image scanning in your pipeline, you need the following: - -- Cluster Image Scanning runs in the `test` stage, which is available by default. If you redefine the stages - in the `.gitlab-ci.yml` file, the `test` stage is required. -- [GitLab Runner](https://docs.gitlab.com/runner/) - with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) - or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) - executor on Linux/amd64. -- Docker `18.09.03` or later installed on the same computer as the runner. If you're using the - shared runners on GitLab.com, then this is already the case. -- [Starboard Operator](https://aquasecurity.github.io/starboard/v0.10.3/operator/installation/kubectl/) - installed and configured in your cluster. -- The configuration for accessing your Kubernetes cluster stored in the `CIS_KUBECONFIG` - [configuration variable](#cicd-variables-for-cluster-image-scanning) - with the type set to `File` (see [Configuring the cluster](#configuring-the-cluster)). - -### Configuring the cluster - -1. Create a new service account. - - To properly fetch vulnerabilities from the cluster and to limit analyzer access to the workload, - you must create a new service account with the cluster role limited to `get`, `list`, and `watch` - `vulnerabilityreports` in the Kubernetes cluster: - - ```shell - kubectl apply -f https://gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning/-/raw/main/gitlab-vulnerability-viewer-service-account.yaml - ``` - -1. Obtain the Kubernetes API URL. - - Get the API URL by running this command: - - ```shell - API_URL=$(kubectl cluster-info | grep -E 'Kubernetes master|Kubernetes control plane' | awk '/http/ {print $NF}') - ``` - -1. Obtain the CA certificate: - - 1. List the secrets with `kubectl get secrets`. One should have a name similar to - `default-token-xxxxx`. Copy that token name for use below. - - 1. Run this command to get the certificate: - - ```shell - CA_CERTIFICATE=$(kubectl get secret <secret name> -o jsonpath="{['data']['ca\.crt']}") - ``` - -1. Obtain the service account token: - - ```shell - TOKEN=$(kubectl -n kube-system get secret $(kubectl -n kube-system get secret | grep gitlab-vulnerability-viewer | awk '{print $1}') -o jsonpath="{.data.token}" | base64 --decode) - ``` - -1. Generate the value for the `CIS_KUBECONFIG` variable. Copy the printed value from the output: - - ```shell - echo " - --- - apiVersion: v1 - kind: Config - clusters: - - name: gitlab-vulnerabilities-viewer - cluster: - server: $API_URL - certificate-authority-data: $CA_CERTIFICATE - contexts: - - name: gitlab-vulnerabilities-viewer - context: - cluster: gitlab-vulnerabilities-viewer - namespace: default - user: gitlab-vulnerabilities-viewer - current-context: gitlab-vulnerabilities-viewer - users: - - name: gitlab-vulnerabilities-viewer - user: - token: $TOKEN - " - ``` - -1. Set the CI/CD variable: - - 1. Navigate to your project's **Settings > CI/CD**. - - 1. Expand the **Variables** section. - - 1. Select **Add variable** and fill in the details: - - - **Key**: `CIS_KUBECONFIG`. - - **Value**: `generated value` - - **Type**: `File` - -WARNING: -The `CIS_KUBECONFIG` variable is accessible by all jobs executed for your project. Mark the -`Protect variable` flag to export this variable to pipelines running on protected branches and tags -only. You can apply additional protection to your cluster by -[restricting service account access to a single namespace](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), -and [configuring Starboard Operator](https://aquasecurity.github.io/starboard/v0.10.3/operator/configuration/#install-modes) -to install in restricted mode. - -### Configuration - -To include the `Cluster-Image-Scanning.gitlab-ci.yml` template (GitLab 14.1 and later), add the -following to your `.gitlab-ci.yml` file: - -```yaml -include: - - template: Security/Cluster-Image-Scanning.gitlab-ci.yml -``` - -The included template: - -- Creates a `cluster_image_scanning` job in your CI/CD pipeline. -- Connects to your Kubernetes cluster with credentials provided in the `CIS_KUBECONFIG` variable and - fetches vulnerabilities found by [Starboard Operator](https://aquasecurity.github.io/starboard/v0.10.3/operator/). - -GitLab saves the results as a -[Cluster Image Scanning report artifact](../../../ci/yaml/artifacts_reports.md#artifactsreportscluster_image_scanning) -that you can download and analyze later. When downloading, you always receive the most recent -artifact. - -#### Customize the cluster image scanning settings - -You can customize how GitLab scans your cluster. For example, to restrict the analyzer to get -results for only a certain workload, use the [`variables`](../../../ci/yaml/index.md#variables) -parameter in your `.gitlab-ci.yml` to set [CI/CD variables](#cicd-variables-for-cluster-image-scanning). -The variables you set in your `.gitlab-ci.yml` overwrite those in -`Cluster-Image-Scanning.gitlab-ci.yml`. - -##### CI/CD variables for cluster image scanning - -You can [configure](#customize-the-cluster-image-scanning-settings) analyzers by using the following CI/CD variables: - -| CI/CD Variable | Default | Description | -| ------------------------------ | ------------- | ----------- | -| `CIS_KUBECONFIG` | `""` | File used to configure access to the Kubernetes cluster. See the [Kubernetes documentation](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/) for more details. | -| `CIS_CONTAINER_NAMES` | `""` | A comma-separated list of container names used in the Kubernetes resources you want to filter vulnerabilities for. For example, `alpine,postgres`. | -| `CIS_RESOURCE_NAMES` | `""` | A comma-separated list of Kubernetes resources you want to filter vulnerabilities for. For example, `nginx,redis`. | -| `CIS_RESOURCE_NAMESPACES` | `""` | A comma-separated list of namespaces of the Kubernetes resources you want to filter vulnerabilities for. For example, `production,staging`. | -| `CIS_RESOURCE_KINDS` | `""` | A comma-separated list of the kinds of Kubernetes resources to filter vulnerabilities for. For example, `deployment,pod`. | -| `CIS_CLUSTER_IDENTIFIER` | `""` | ID of the Kubernetes cluster integrated with GitLab. This is used to map vulnerabilities to the cluster so they can be filtered in the Vulnerability Report page. | -| `CIS_CLUSTER_AGENT_IDENTIFIER` | `""` | ID of the Kubernetes cluster agent integrated with GitLab. This maps vulnerabilities to the agent so they can be filtered in the Vulnerability Report page. | - -#### Override the cluster image scanning template - -If you want to override the job definition (for example, to change properties like `variables`), you -must declare and override a job after the template inclusion, and then -specify any additional keys. - -This example sets `CIS_RESOURCE_NAME` to `nginx`: - -```yaml -include: - - template: Security/Cluster-Image-Scanning.gitlab-ci.yml - -cluster_image_scanning: - variables: - CIS_RESOURCE_NAME: nginx -``` - -#### Connect with Kubernetes cluster associated to the project - -If you want to connect to the Kubernetes cluster associated with the project and run Cluster Image Scanning jobs without -configuring the `CIS_KUBECONFIG` variable, you must extend `cluster_image_scanning` and specify the environment you want to scan. - -This example configures the `cluster_image_scanning` job to scan the Kubernetes cluster connected with the `staging` environment: - -```yaml -cluster_image_scanning: - environment: - name: staging - action: prepare -``` - -### Reports JSON format - -The cluster image scanning tool emits a JSON report file. For more information, see the -[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/container-scanning-report-format.json). - -Here's an example cluster image scanning report: - -```json-doc -{ - "version": "14.0.2", - "scan": { - "scanner": { - "id": "starboard_trivy", - "name": "Trivy (using Starboard Operator)", - "url": "https://github.com/aquasecurity/starboard", - "vendor": { - "name": "GitLab" - }, - "version": "0.16.0" - }, - "start_time": "2021-04-28T12:47:00Z", - "end_time": "2021-04-28T12:47:00Z", - "type": "cluster_image_scanning", - "status": "success" - }, - "vulnerabilities": [ - { - "id": "c15f22205ee842184c2d55f1a207b3708283353f85083d66c34379c709b0ac9d", - "category": "cluster_image_scanning", - "message": "CVE-2011-3374 in apt", - "description": "", - "cve": "library/nginx:1.18:apt:CVE-2011-3374", - "severity": "Low", - "confidence": "Unknown", - "solution": "Upgrade apt from 1.8.2.2", - "scanner": { - "id": "starboard_trivy", - "name": "Trivy (using Starboard Operator)" - }, - "location": { - "dependency": { - "package": { - "name": "apt" - }, - "version": "1.8.2.2" - }, - "operating_system": "library/nginx:1.18", - "image": "index.docker.io/library/nginx:1.18" - }, - "identifiers": [ - { - "type": "cve", - "name": "CVE-2011-3374", - "value": "CVE-2011-3374", - "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2011-3374" - } - ], - "links": [ - "https://avd.aquasec.com/nvd/cve-2011-3374" - ] - } - ] -} -``` - -<!--- end_remove --> -## Cluster image scanning with the GitLab agent - -You can use the [GitLab agent](../../clusters/agent/index.md) to -scan images from within your Kubernetes cluster and record the vulnerabilities in GitLab. - -### Prerequisites - -- [GitLab agent](../../clusters/agent/install/index.md) - set up in GitLab, installed in your cluster, and configured using a configuration repository. - -### Configuration - -The agent runs the cluster image scanning once the `starboard` -directive is added to your [agent's configuration repository](../../clusters/agent/vulnerabilities.md). - -## Security Dashboard - -The [Security Dashboard](../security_dashboard/index.md) shows you an overview of all -the security vulnerabilities in your groups, projects, and pipelines. - -## Interacting with the vulnerabilities - -After you find a vulnerability, you can address it in the [vulnerability report](../vulnerabilities/index.md) -or the [GitLab agent's](../../clusters/agent/vulnerabilities.md) -details section. -<!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> - -## Troubleshooting - -### Getting warning message `gl-cluster-image-scanning-report.json: no matching files` - -For information on this error, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). - -<!--- end_remove --> +<!-- This redirect file can be deleted after <2022-08-19>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index 61a2121b9c6..09292dcb92b 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -52,8 +52,8 @@ You can configure the following security controls: - Select **Configure with a merge request** to create a merge request with the changes required to enable Container Scanning. For more details, see [Enable Container Scanning through an automatic merge request](../container_scanning/index.md#enable-container-scanning-through-an-automatic-merge-request). -- [Cluster Image Scanning](../cluster_image_scanning/index.md) - - Can be configured with `.gitlab-ci.yml`. For more details, read [Cluster Image Scanning](../../../user/application_security/cluster_image_scanning/#configuration). +- [Operational Container Scanning](../../clusters/agent/vulnerabilities.md) + - Can be configured by adding a configuration block to your agent configuration. For more details, read [Operational Container Scanning](../../clusters/agent/vulnerabilities.md#enable-cluster-vulnerability-scanning). - [Secret Detection](../secret_detection/index.md) - Select **Configure with a merge request** to create a merge request with the changes required to enable Secret Detection. For more details, read [Enable Secret Detection via an automatic merge request](../secret_detection/index.md#enable-secret-detection-via-an-automatic-merge-request). diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 2828b56a5d1..c41385a3569 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -260,7 +260,7 @@ Support depends on which scanner is used: | Distribution | Grype | Trivy | | -------------- | ----- | ----- | | Alma Linux | | ✅ | -| Alpine Linux | ✅ | | +| Alpine Linux | ✅ | ✅ | | Amazon Linux | ✅ | ✅ | | BusyBox | ✅ | | | CentOS | ✅ | ✅ | @@ -450,7 +450,7 @@ To allowlist specific vulnerabilities, follow these steps: the format described in [`vulnerability-allowlist.yml` data format](#vulnerability-allowlistyml-data-format). 1. Add the `vulnerability-allowlist.yml` file to the root folder of your project's Git repository. -#### vulnerability-allowlist.yml data format +#### `vulnerability-allowlist.yml` data format The `vulnerability-allowlist.yml` file is a YAML file that specifies a list of CVE IDs of vulnerabilities that are **allowed** to exist, because they're _false positives_, or they're _not applicable_. diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md index 14e98766f0f..b2b7dd85468 100644 --- a/doc/user/application_security/coverage_fuzzing/index.md +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -145,7 +145,7 @@ You can download the JSON report file from the CI/CD pipelines page. For more in ## Corpus registry > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5017) in GitLab 14.8. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/347187) in GitLab 14.9. [Feature flags `corpus_management` and `corpus_management_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/328418) removed. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/347187) in GitLab 14.9. [Feature flags `corpus_management` and `corpus_management_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/328418) removed. The corpus registry is a library of corpuses. Corpuses in a project's registry are available to all jobs in that project. A project-wide registry is a more efficient way to manage corpuses than @@ -313,6 +313,10 @@ This creates two jobs: The `covfuzz-ci.yml` is the same as that in the [original synchronous example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example#running-go-fuzz-from-ci). +## FIPS-enabled binary + +[Starting in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/352549) the coverage fuzzing binary is compiled with `golang-fips` on Linux x86 and uses OpenSSL as the cryptographic backend. For more details, see [FIPS compliance at GitLab with Go](../../../development/fips_compliance.md#go). + ## Offline environment To use coverage fuzzing in an offline environment: diff --git a/doc/user/application_security/dast/browser_based.md b/doc/user/application_security/dast/browser_based.md index 4cde1847419..ffcd496e2c3 100644 --- a/doc/user/application_security/dast/browser_based.md +++ b/doc/user/application_security/dast/browser_based.md @@ -111,7 +111,7 @@ a page fully loaded. Browser-based scans consider a page loaded when: 1. The [DOMContentLoaded](https://developer.mozilla.org/en-US/docs/Web/API/Window/DOMContentLoaded_event) event has fired. 1. There are no open or outstanding requests that are deemed important, such as JavaScript and CSS. Media files are usually deemed unimportant. 1. Depending on whether the browser executed a navigation, was forcibly transitioned, or action: - + - There are no new Document Object Model (DOM) modification events after the `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT`, `DAST_BROWSER_STABILITY_TIMEOUT`, or `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` durations. After these events have occurred, browser-based scans consider the page loaded and ready, and attempt the next action. diff --git a/doc/user/application_security/dast/checks/1004.1.md b/doc/user/application_security/dast/checks/1004.1.md index 9626973eb36..72af1156b95 100644 --- a/doc/user/application_security/dast/checks/1004.1.md +++ b/doc/user/application_security/dast/checks/1004.1.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w The {cookie_name} cookie was transmitted in a `Set-Cookie` header without the `HttpOnly` attribute set. To prevent JavaScript being able to access the cookie value - usually via `document.cookies` - all -cookies that are used for authorization or contain sensitive information should have the `HttpOnly` attribute +cookies that are used for authorization should have the `HttpOnly` attribute set. ## Remediation diff --git a/doc/user/application_security/dast/checks/16.7.md b/doc/user/application_security/dast/checks/16.7.md new file mode 100644 index 00000000000..a02fb3a451f --- /dev/null +++ b/doc/user/application_security/dast/checks/16.7.md @@ -0,0 +1,42 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Strict-Transport-Security header missing or invalid + +## Description + +The `Strict-Transport-Security` header was found to be missing or invalid. The `Strict-Transport-Security` +header allows web site operators to force communications to occur over a TLS connection. By enabling this +header, websites can protect their users from various forms of network eavesdropping or interception attacks. +While most browsers prevent mixed-content (loading resources from HTTP when navigating from an HTTPS site), +this header also ensures that all resource requests are only ever initiated over a secure transport. + +## Remediation + +Only three directives are applicable for the `Strict-Transport-Security` header. + +1. `max-age`: This required directive specifies how long (in seconds) after receiving the response it should communicate only over a secure transport. +1. `includeSubDomains`: This optional, valueless directive signals that the policy applies to this host as well as any subdomains found under this host's domain. +1. `preload`: While not part of the specification, setting this optional value allows major browser organizations to add this site into the browser's preloaded set of HTTPS sites. This requires further action on behalf of the website operator to submit their domain to the browser's HSTS preload list. See [hstspreload.org](https://hstspreload.org/) for more information. + +Note that invalid directives, or the `Strict-Transport-Security` header appearing more than once (if the values are +different) is considered invalid. + +Prior to adding to this security configuration to your website, it is recommended you review the hstspreload.org [Deployment +Recommendations](https://hstspreload.org/#deployment-recommendations). + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 16.7 | true | 16 | Passive | Low | + +## Links + +- [CWE](https://cwe.mitre.org/data/definitions/16.html) +- [Deployment Recommendations](https://hstspreload.org/#deployment-recommendations) +- [OWASP](https://cheatsheetseries.owasp.org/cheatsheets/HTTP_Strict_Transport_Security_Cheat_Sheet.html) +- [RFC](https://datatracker.ietf.org/doc/html/rfc6797) diff --git a/doc/user/application_security/dast/checks/200.1.md b/doc/user/application_security/dast/checks/200.1.md index 9795ad11b0b..fcd329c3f2b 100644 --- a/doc/user/application_security/dast/checks/200.1.md +++ b/doc/user/application_security/dast/checks/200.1.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Description -A private RFC 1918 was identified in the target application. Public facing websites should not be issuing +A private RFC 1918/RFC 4193 address was identified in the target application. Public facing websites should not be issuing requests to private IP Addresses. Attackers attempting to execute subsequent attacks, such as Server-Side Request Forgery (SSRF), may be able to use this information to identify additional internal targets. @@ -27,3 +27,4 @@ facing version, or remove the reference from the target application. - [CWE](https://cwe.mitre.org/data/definitions/200.html) - [RFC](https://datatracker.ietf.org/doc/html/rfc1918) +- [RFC](https://datatracker.ietf.org/doc/html/rfc4193) diff --git a/doc/user/application_security/dast/checks/209.1.md b/doc/user/application_security/dast/checks/209.1.md new file mode 100644 index 00000000000..2e4163bdec0 --- /dev/null +++ b/doc/user/application_security/dast/checks/209.1.md @@ -0,0 +1,43 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Generation of error message containing sensitive information + +## Description + +The application was found to return error data such as stack traces. Depending on the data contained within the error message, +this information could be used by an attacker to conduct further attacks. While stack traces are helpful during development +and debugging, they should not be presented to users when an error occurs. + +## Remediation + +Applications should handle exception conditions internally and map known failure types to error codes that can be displayed +to a user. These error codes should be customized to the application and returned along with the relevant HTTP error code. + +When an error occurs, the application identifies the error type or class, and displays a numerical value to the +user. Requests should also be tracked so when a user is presented with an error code, it has a corresponding request ID. +Support teams can then correlate the HTTP error, the customized error code, and the request ID in the log files to +determine the root cause of the error without leaking details to the end user. + +Example of returning customized errors: + +```plaintext +HTTP/1.1 500 Internal Server Error +... +Error [0004] Occurred, please contact support or re-try your request again shortly. +Request ID [a4bc91def12] +... +``` + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 209.1 | false | 209 | Passive | Low | + +## Links + +- [CWE](https://cwe.mitre.org/data/definitions/209.html) diff --git a/doc/user/application_security/dast/checks/319.1.md b/doc/user/application_security/dast/checks/319.1.md new file mode 100644 index 00000000000..d598fb70ce3 --- /dev/null +++ b/doc/user/application_security/dast/checks/319.1.md @@ -0,0 +1,37 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Mixed Content + +## Description + +The target application was found to request resources over insecure transport protocols (HTTP). This is usually due to HTML +elements which load resources using the `http://` scheme instead of `https://`. It should be noted that most modern browsers +block these requests automatically so there is limited risk. + +Some parts of the application may not behave correctly since these files are not being properly loaded. + +## Remediation + +Ensure all HTML elements which load resources from a URL (JavaScript, stylesheets, images, video and other media) are set to +use the `https://` scheme instead of `http://`. Alternatively, developers may use the `//` scheme, which will only load resources +over the same protocol that the originating page was loaded. + +A browser visiting the website `https://example.com` with the HTML loading a file using +`<script src="//example.com/cdn/bundle.js"></script>`, would ensure the `example.com/cdn/bundle.js` file was loaded over +HTTPS. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 319.1 | true | 319 | Passive | Info | + +## Links + +- [OWASP](https://owasp.org/www-community/vulnerabilities/Insecure_Transport) +- [CWE](https://cwe.mitre.org/data/definitions/319.html) +- [MDN](https://developer.mozilla.org/en-US/docs/Web/Security/Mixed_content) diff --git a/doc/user/application_security/dast/checks/352.1.md b/doc/user/application_security/dast/checks/352.1.md new file mode 100644 index 00000000000..4daba908331 --- /dev/null +++ b/doc/user/application_security/dast/checks/352.1.md @@ -0,0 +1,41 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Absence of anti-CSRF tokens + +## Description + +The application failed to protect against Cross-Site Request Forgery (CSRF) by using +secure application tokens or `SameSite` cookie directives. + +The vulnerability can be exploited by an attacker creating a link or form on a third +party site and tricking an authenticated victim to access them. + +## Remediation + +Consider setting all session cookies to have the `SameSite=Strict` attribute. However, +it should be noted that this may impact usability when sharing links across other mediums. +It is recommended that a two cookie based approach is taken, as outlined in the +[Top level navigations](https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-rfc6265bis-08#section-8.8.2) section +of the RFC. + +If the application is using a common framework, there is a chance that Anti-CSRF protection +is built in but needs to be enabled. Consult your application framework documentation for +details. + +If neither of the above are applicable, it is **strongly** recommended that a third party library is used. +Implementing a secure Anti-CSRF system is a significant investment and difficult to do correctly. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 352.1 | true | 352 | Passive | Medium | + +## Links + +- [OWASP](https://owasp.org/www-community/attacks/csrf) +- [CWE](https://cwe.mitre.org/data/definitions/352.html) diff --git a/doc/user/application_security/dast/checks/359.1.md b/doc/user/application_security/dast/checks/359.1.md index af1fdf8a596..076ab2da0d5 100644 --- a/doc/user/application_security/dast/checks/359.1.md +++ b/doc/user/application_security/dast/checks/359.1.md @@ -8,8 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Description -The target application was found to return credit card information in the response. Organizations -found returning such information may be in violation of industry regulations and could face fines. +The target application was found to return credit card information in the response. Organizations +found returning such information may be in violation of industry regulations and could face fines. ## Remediation @@ -17,7 +17,7 @@ PII such as credit cards should never be directly returned to the user. The majo the last few digits or characters of the identifier. For example, credit card numbers should only return the last four digits: `****-****-****-1234`. Ensure this masking is done on the server and only then send the masked data back to the client. Do not rely on client side JavaScript or other methods -to mask these values as the data could still be intercepted or unmasked. +to mask these values as the data could still be intercepted or unmasked. Additionally, credit card information should never be stored un-encrypted in files or databases. diff --git a/doc/user/application_security/dast/checks/359.2.md b/doc/user/application_security/dast/checks/359.2.md index beb99e26097..2c59b5e321f 100644 --- a/doc/user/application_security/dast/checks/359.2.md +++ b/doc/user/application_security/dast/checks/359.2.md @@ -8,16 +8,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Description -The target application was found to return social security number (SSN) information in the response. Organizations -found returning such information may be in violation of (United States) state or federal laws and may face stiff penalties. +The target application was found to return social security number (SSN) information in the response. Organizations +found returning such information may be in violation of (United States) state or federal laws and may face stiff penalties. ## Remediation -PII such as social security numbers should never be directly returned to the user. The majority of the information +PII such as social security numbers should never be directly returned to the user. The majority of the information should masked except the last few digits or characters of the identifier. For example, social security numbers only be displayed with the last four digits: `***-**-1234`. Ensure this masking is done on the server and only then send the masked data back to the client. Do not rely on client side JavaScript or other methods -to mask these values as the data could still be intercepted or unmasked. +to mask these values as the data could still be intercepted or unmasked. Additionally, social security numbers should never be stored un-encrypted in files or databases. diff --git a/doc/user/application_security/dast/checks/548.1.md b/doc/user/application_security/dast/checks/548.1.md index d6371c5491d..1da2ce58247 100644 --- a/doc/user/application_security/dast/checks/548.1.md +++ b/doc/user/application_security/dast/checks/548.1.md @@ -39,7 +39,7 @@ indexing. ## Links -- [CWE](https://cwe.mitre.org/data/definitions/598.html) +- [CWE](https://cwe.mitre.org/data/definitions/548.html) - [Apache Options](https://httpd.apache.org/docs/2.4/mod/core.html#options) - [NGINX autoindex](https://nginx.org/en/docs/http/ngx_http_autoindex_module.html) - [IIS directoryBrowse element](https://docs.microsoft.com/en-us/iis/configuration/system.webserver/directorybrowse) diff --git a/doc/user/application_security/dast/checks/598.2.md b/doc/user/application_security/dast/checks/598.2.md index f6c6787128d..05d04b71cf0 100644 --- a/doc/user/application_security/dast/checks/598.2.md +++ b/doc/user/application_security/dast/checks/598.2.md @@ -16,7 +16,7 @@ be able to gain access to the target account. ## Remediation Passwords should never be sent in GET requests. When authenticating users or requesting users -reset their passwords, always use POST requests to transmit sensitive data. +reset their passwords, always use `POST` requests to transmit sensitive data. ## Details diff --git a/doc/user/application_security/dast/checks/598.3.md b/doc/user/application_security/dast/checks/598.3.md index fa6fdf43e1c..be17fdcaef6 100644 --- a/doc/user/application_security/dast/checks/598.3.md +++ b/doc/user/application_security/dast/checks/598.3.md @@ -17,7 +17,7 @@ target account. ## Remediation Authorization header details should never be sent in GET requests. When transmitting sensitive information -such as JWT tokens, always use POST requests or headers to transmit the sensitive data. +such as JWT tokens, always use `POST` requests or headers to transmit the sensitive data. ## Details diff --git a/doc/user/application_security/dast/checks/601.1.md b/doc/user/application_security/dast/checks/601.1.md new file mode 100644 index 00000000000..26ccd877104 --- /dev/null +++ b/doc/user/application_security/dast/checks/601.1.md @@ -0,0 +1,34 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# URL redirection to untrusted site ('open redirect') + +## Description + +This site was found to allow open redirects from user supplied input. Open redirects are commonly +abused in phishing attacks where the original domain or URL looks like a legitimate link, but then +redirects a user to a malicious site. An example would be +`https://example.com/redirect?url=https://%62%61%64%2e%63%6f%6d%2f%66%61%6b%65%6c%6f%67%69%6e` which, +when decoded turns into `bad.com/fakelogin`. + +## Remediation + +Never redirect a client based on user input found in a `GET` request. It is recommended that the list +of target links to redirect a user to are contained server side, and retrieved using a numerical value +as an index to return the link to be redirected to. For example, `/redirect?id=1` would cause the +application to look up the `1` index and return a URL such as `https://example.com`. This URL would +then be used to redirect the user, using the 301 response code and `Location` header. + +## Details + +| ID | Aggregated | CWE | Type | Risk | +|:---|:--------|:--------|:--------|:--------| +| 601.1 | true | 601 | Passive | Low | + +## Links + +- [OWASP](https://owasp.org/www-project-cheat-sheets/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) +- [CWE](https://cwe.mitre.org/data/definitions/601.html) diff --git a/doc/user/application_security/dast/checks/index.md b/doc/user/application_security/dast/checks/index.md index 629ff1c3a8d..e2947d5b120 100644 --- a/doc/user/application_security/dast/checks/index.md +++ b/doc/user/application_security/dast/checks/index.md @@ -17,13 +17,18 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [16.4](16.4.md) | X-Backend-Server header exposes server information | Info | Passive | | [16.5](16.5.md) | AspNet header exposes version information | Low | Passive | | [16.6](16.6.md) | AspNetMvc header exposes version information | Low | Passive | +| [16.7](16.7.md) | Strict-Transport-Security header missing or invalid | Low | Passive | | [200.1](200.1.md) | Exposure of sensitive information to an unauthorized actor (private IP address) | Low | Passive | +| [209.1](209.1.md) | Generation of error message containing sensitive information | Low | Passive | +| [319.1](319.1.md) | Mixed Content | Info | Passive | +| [352.1](352.1.md) | Absence of anti-CSRF tokens | Medium | Passive | | [359.1](359.1.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (credit card) | Medium | Passive | | [359.2](359.2.md) | Exposure of Private Personal Information (PII) to an unauthorized actor (United States social security number) | Medium | Passive | | [548.1](548.1.md) | Exposure of information through directory listing | Low | Passive | | [598.1](598.1.md) | Use of GET request method with sensitive query strings (session ID) | Medium | Passive | | [598.2](598.2.md) | Use of GET request method with sensitive query strings (password) | Medium | Passive | | [598.3](598.3.md) | Use of GET request method with sensitive query strings (Authorization header details) | Medium | Passive | +| [601.1](601.1.md) | URL redirection to untrusted site ('open redirect') | Low | Passive | | [614.1](614.1.md) | Sensitive cookie without Secure attribute | Low | Passive | | [693.1](693.1.md) | Missing X-Content-Type-Options: nosniff | Low | Passive | | [829.1](829.1.md) | Inclusion of Functionality from Untrusted Control Sphere | Low | Passive | diff --git a/doc/user/application_security/dast/dast_troubleshooting.md b/doc/user/application_security/dast/dast_troubleshooting.md index 9969526c906..50570b89920 100644 --- a/doc/user/application_security/dast/dast_troubleshooting.md +++ b/doc/user/application_security/dast/dast_troubleshooting.md @@ -89,6 +89,16 @@ include: - template: DAST.latest.gitlab-ci.yml ``` +## Getting error `shell not found` when using DAST CI/CD template + +When including the DAST CI/CD template as described in the documentation, the job may fail, with an error like the following recorded in the job logs: + +```shell +shell not found +``` + +To avoid this error, make sure you are using the latest stable version of Docker. More information is available in [issue 358847](https://gitlab.com/gitlab-org/gitlab/-/issues/358847). + ## Lack of IPv6 support Due to the underlying [ZAProxy engine not supporting IPv6](https://github.com/zaproxy/zaproxy/issues/3705), DAST is unable to scan or crawl IPv6-based applications. diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 1389db65713..25b4b705025 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -178,7 +178,8 @@ To enable DAST to run automatically, either: #### Include the DAST template -> This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62597) to DAST_VERSION: 2 in GitLab 14.0. +> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62597) to DAST_VERSION: 2 in GitLab 14.0. +> - This template was [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87183) to DAST_VERSION: 3 in GitLab 15.0. If you want to manually add DAST to your application, the DAST job is defined in a CI/CD template file. Updates to the template are provided with GitLab @@ -333,7 +334,7 @@ Vulnerability rules in an API scan are different than those in a normal website A new DAST API scanning engine is available in GitLab 13.12 and later. For more details, see [DAST API scanning engine](../dast_api). The new scanning engine supports REST, SOAP, GraphQL, and generic APIs using forms, XML, and JSON. Testing can be performed using OpenAPI, Postman Collections, and HTTP Archive (HAR) documents. -The target API instance’s base URL is provided by using the `DAST_API_TARGET_URL` variable or an `environment_url.txt` file. +The target API instance's base URL is provided by using the `DAST_API_TARGET_URL` variable or an `environment_url.txt` file. #### Specification format @@ -493,7 +494,7 @@ To perform a [full scan](#full-scan) on the listed paths, use the `DAST_FULL_SCA ### List URLs scanned When DAST completes scanning, the merge request page states the number of URLs scanned. -Click **View details** to view the web console output which includes the list of scanned URLs. +Select **View details** to view the web console output which includes the list of scanned URLs.  @@ -574,7 +575,7 @@ DAST scan with both configured exits with an error. By default, several rules are disabled because they either take a long time to run or frequently generate false positives. The complete list of disabled rules -can be found in [exclude_rules.yml](https://gitlab.com/gitlab-org/security-products/dast/-/blob/main/src/config/exclude_rules.yml). +can be found in [`exclude_rules.yml`](https://gitlab.com/gitlab-org/security-products/dast/-/blob/main/src/config/exclude_rules.yml). The lists for `DAST_EXCLUDE_RULES` and `DAST_ONLY_INCLUDE_RULES` **must** be enclosed in double quotes (`"`), otherwise they are interpreted as numeric values. @@ -737,7 +738,7 @@ by the application as correctly authenticated. Authentication supports single form logins, multi-step login forms, and authenticating to URLs outside of the configured target URL. WARNING: -**NEVER** run an authenticated scan against a production server. When an authenticated +**Never** run an authenticated scan against a production server. When an authenticated scan is run, it may perform *any* function that the authenticated user can. This includes actions like modifying and deleting data, submitting forms, and following links. Only run an authenticated scan against a test server. @@ -846,7 +847,7 @@ Many web applications show the user the login form in a pop-up (modal) window. For these applications, navigating to the form requires both: - A starting URL. -- A list of elements to click to display the modal window. +- A list of elements to select to display the modal window. When `DAST_BROWSER_PATH_TO_LOGIN_FORM` is present, like in this example: @@ -1327,7 +1328,7 @@ class DastWebsiteTargetView(View): ##### Node (with Express) example for on-demand scan Here's how you can add a -[custom header in Node (with Express)](http://expressjs.com/en/5x/api.html#res.append): +[custom header in Node (with Express)](https://expressjs.com/en/5x/api.html#res.append): ```javascript app.get('/dast-website-target', function(req, res) { @@ -1399,17 +1400,19 @@ and DAST site profiles are included in the [audit log](../../../administration/a ## Reports -The DAST tool outputs a report file in JSON format by default. However, this tool can also generate reports in -Markdown, HTML, and XML. For more information, see the [schema for DAST reports](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dast-report-format.json). +The DAST tool outputs a `gl-dast-report.json` report file containing details of the scan and its results. +This file is included in the job's artifacts. JSON is the default format, but +you can output the report in Markdown, HTML, and XML formats. To specify an alternative +format, use a [CI/CD variable](#available-cicd-variables). You can also use a CI/CD variable +to configure the job to output the `gl-dast-debug-auth-report.html` file which helps when debugging +authentication issues. -### JSON +For details of the report's schema, see the [schema for DAST reports](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/dast-report-format.json). Example reports can be found in the +[DAST repository](https://gitlab.com/gitlab-org/security-products/dast/-/tree/main/test/end-to-end/expect). WARNING: -The JSON report artifacts are not a public API of DAST and their format is expected to change in the future. - -The DAST tool always emits a JSON report file called `gl-dast-report.json` and -sample reports can be found in the -[DAST repository](https://gitlab.com/gitlab-org/security-products/dast/-/tree/main/test/end-to-end/expect). +The JSON report artifacts are not a public API of DAST and their format is expected to change in the +future. ## Optimizing DAST diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index a1b19c52b20..9128576bf29 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -538,7 +538,7 @@ can be added, removed, and modified by creating a custom configuration. | CI/CD variable | Description | |------------------------------------------------------|--------------------| | `SECURE_ANALYZERS_PREFIX` | Specify the Docker registry base address from which to download the analyzer. | -| `DAST_API_VERSION` | Specify DAST API container version. Defaults to `1`. | +| `DAST_API_VERSION` | Specify DAST API container version. Defaults to `2`. | | `DAST_API_IMAGE_SUFFIX` | Specify a container image suffix. Defaults to none. | | `DAST_API_TARGET_URL` | Base URL of API testing target. | |[`DAST_API_CONFIG`](#configuration-files) | DAST API configuration file. Defaults to `.gitlab-dast-api.yml`. | @@ -837,7 +837,7 @@ Adding some basic logging to your overrides script is useful in case the script Following our example, we provided `renew_token.py` in the environment variable `DAST_API_OVERRIDES_CMD`. Please notice two things in the script: - Log file is saved in the location indicated by the environmental variable `CI_PROJECT_DIR`. -- Log file name should match `gl-*.log`. +- Log filename should match `gl-*.log`. ```python #!/usr/bin/env python @@ -1021,6 +1021,19 @@ variables: DAST_API_EXCLUDE_PATHS=/auth*;/v1/* ``` +To exclude one or more nested levels within a path we use `**`. In this example we are testing API endpoints. We are testing `/api/v1/` and `/api/v2/` of a data query requesting `mass`, `brightness` and `coordinates` data for `planet`, `moon`, `star`, and `satellite` objects. Example paths that could be scanned include, but are not limited to: + +- `/api/v2/planet/coordinates` +- `/api/v1/star/mass` +- `/api/v2/satellite/brightness` + +In this example we test the `brightness` endpoint only: + +```yaml +variables: + DAST_API_EXCLUDE_PATHS=/api/**/mass;/api/**/coordinates +``` + ### Exclude parameters > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292196) in GitLab 14.10. @@ -1250,7 +1263,7 @@ variables: The `dast-api-exclude-parameters.json` is a JSON document that follows the structure of [exclude parameters document](#exclude-parameters-using-a-json-document). -### Exclude URLS +### Exclude URLs > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357195) in GitLab 14.10. @@ -1332,12 +1345,12 @@ Follow these steps to view details of a vulnerability: - In a project, go to the project's **{shield}** **Security & Compliance > Vulnerability Report** page. This page shows all vulnerabilities from the default branch only. - - In a merge request, go the merge request's **Security** section and click the **Expand** + - In a merge request, go the merge request's **Security** section and select the **Expand** button. DAST API vulnerabilities are available in a section labeled - **DAST detected N potential vulnerabilities**. Click the title to display the vulnerability + **DAST detected N potential vulnerabilities**. Select the title to display the vulnerability details. -1. Click the vulnerabilities title to display the details. The table below describes these details. +1. Select the vulnerabilities title to display the details. The table below describes these details. | Field | Description | |:--------------------|:----------------------------------------------------------------------------------------| @@ -1489,14 +1502,14 @@ Steps: The Docker image for DAST API must be pulled (downloaded) from the public registry and then pushed (imported) into a local registry. The GitLab container registry can be used to locally host the Docker image. This process can be performed using a special template. See [loading Docker images onto your offline host](../offline_deployments/index.md#loading-docker-images-onto-your-offline-host) for instructions. -Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-fuzzing:1` results in a valid image location. +Once the Docker image is hosted locally, the `SECURE_ANALYZERS_PREFIX` variable is set with the location of the local registry. The variable must be set such that concatenating `/api-security:2` results in a valid image location. NOTE: -DAST API and API Fuzzing both use the same underlying Docker image `api-fuzzing:1`. +DAST API and API Fuzzing both use the same underlying Docker image `api-security:2`. -For example, the below line sets a registry for the image `registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:1`: +For example, the below line sets a registry for the image `registry.gitlab.com/security-products/api-security:2`: -`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"` +`SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products"` NOTE: Setting `SECURE_ANALYZERS_PREFIX` changes the Docker image registry location for all GitLab Secure templates. diff --git a/doc/user/application_security/dependency_list/index.md b/doc/user/application_security/dependency_list/index.md index 78de740c96d..03c97c85dbc 100644 --- a/doc/user/application_security/dependency_list/index.md +++ b/doc/user/application_security/dependency_list/index.md @@ -13,7 +13,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Use the dependency list to review your project's dependencies and key details about those dependencies, including their known vulnerabilities. It is a collection of dependencies in your project, including existing and new findings. -To see the dependency list, go to your project and select **Security & Compliance > Dependency List**. +To see the dependency list, go to your project and select **Security & Compliance > Dependency list**. This information is sometimes referred to as a Software Bill of Materials, SBOM, or BOM. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 87689572866..08e2dcd2e7e 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -17,6 +17,24 @@ aspects of inspecting the items your code uses. These items typically include ap dependencies that are almost always imported from external sources, rather than sourced from items you wrote yourself. +If you're using [GitLab CI/CD](../../../ci/index.md), you can use dependency scanning to analyze +your dependencies for known vulnerabilities. GitLab scans all dependencies, including transitive +dependencies (also known as nested dependencies). You can take advantage of dependency scanning by +either: + +- [Including the dependency scanning template](#configuration) + in your existing `.gitlab-ci.yml` file. +- Implicitly using + the [auto dependency scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning) + provided by [Auto DevOps](../../../topics/autodevops/index.md). + +GitLab checks the dependency scanning report, compares the found vulnerabilities +between the source and target branches, and shows the information on the +merge request. The results are sorted by the [severity](../vulnerabilities/severities.md) of the +vulnerability. + + + ## Dependency Scanning compared to Container Scanning GitLab offers both Dependency Scanning and Container Scanning @@ -58,26 +76,6 @@ The following table summarizes which types of dependencies each scanning tool ca 1. Lock file must be present in the image to be detected. 1. Binary file must be present in the image to be detected. -## Overview - -If you're using [GitLab CI/CD](../../../ci/index.md), you can use dependency scanning to analyze -your dependencies for known vulnerabilities. GitLab scans all dependencies, including transitive -dependencies (also known as nested dependencies). You can take advantage of dependency scanning by -either: - -- [Including the dependency scanning template](#configuration) - in your existing `.gitlab-ci.yml` file. -- Implicitly using - the [auto dependency scanning](../../../topics/autodevops/stages.md#auto-dependency-scanning) - provided by [Auto DevOps](../../../topics/autodevops/index.md). - -GitLab checks the dependency scanning report, compares the found vulnerabilities -between the source and target branches, and shows the information on the -merge request. The results are sorted by the [severity](../vulnerabilities/severities.md) of the -vulnerability. - - - ## Requirements Dependency Scanning runs in the `test` stage, which is available by default. If you redefine the @@ -95,7 +93,7 @@ is **not** `19.03.0`. See [troubleshooting information](#error-response-from-dae WARNING: Dependency Scanning does not support run-time installation of compilers and interpreters. -If you have need of this, please explain why by filling out the survey [here](https://docs.google.com/forms/d/e/1FAIpQLScKo7xEYA65rOjPTGIufAyfjPGnCALSJZoTxBlvskfFMEOZMw/viewform). +If you need it, please explain why by filling out [the survey](https://docs.google.com/forms/d/e/1FAIpQLScKo7xEYA65rOjPTGIufAyfjPGnCALSJZoTxBlvskfFMEOZMw/viewform). ## Supported languages and package managers @@ -110,6 +108,8 @@ maximum of two directory levels from the repository's root. For example, the `gemnasium-dependency_scanning` job is enabled if a repository contains either `Gemfile`, `api/Gemfile`, or `api/client/Gemfile`, but not if the only supported dependency file is `api/v1/client/Gemfile`. +When a supported dependency file is detected, all dependencies, including transitive dependencies are analyzed. There is no limit to the depth of nested or transitive dependencies that are analyzed. + The following languages and dependency managers are supported: <style> @@ -154,7 +154,7 @@ table.supported-languages ul { <tbody> <tr> <td>Ruby</td> - <td>N/A</td> + <td>Not applicable</td> <td><a href="https://bundler.io/">Bundler</a></td> <td> <ul> @@ -167,7 +167,7 @@ table.supported-languages ul { </tr> <tr> <td>PHP</td> - <td>N/A</td> + <td>Not applicable</td> <td><a href="https://getcomposer.org/">Composer</a></td> <td><code>composer.lock</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -175,7 +175,7 @@ table.supported-languages ul { </tr> <tr> <td>C</td> - <td rowspan="2">N/A</td> + <td rowspan="2">Not applicable</td> <td rowspan="2"><a href="https://conan.io/">Conan</a></td> <td rowspan="2"><a href="https://docs.conan.io/en/latest/versioning/lockfiles.html"><code>conan.lock</code></a></td> <td rowspan="2"><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -186,7 +186,7 @@ table.supported-languages ul { </tr> <tr> <td>Go</td> - <td>N/A</td> + <td>Not applicable</td> <td><a href="https://go.dev/">Go</a></td> <td><code>go.sum</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -194,8 +194,16 @@ table.supported-languages ul { </tr> <tr> <td rowspan="2">Java</td> - <td rowspan="2">8, 11, 13, 14, 15, 16, or 17</td> - <td><a href="https://gradle.org/">Gradle</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-1">1</a></b></sup></td> + <td rowspan="2"> + 8, + 11, + 13<sup><b><a href="#notes-regarding-supported-languages-and-package-managers-1">1</a></b></sup>, + 14<sup><b><a href="#notes-regarding-supported-languages-and-package-managers-1">1</a></b></sup>, + 15<sup><b><a href="#notes-regarding-supported-languages-and-package-managers-1">1</a></b></sup>, + 16<sup><b><a href="#notes-regarding-supported-languages-and-package-managers-1">1</a></b></sup>, + or 17 + </td> + <td><a href="https://gradle.org/">Gradle</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-2">2</a></b></sup></td> <td> <ul> <li><code>build.gradle</code></li> @@ -213,7 +221,7 @@ table.supported-languages ul { </tr> <tr> <td rowspan="2">JavaScript</td> - <td>N/A</td> + <td>Not applicable</td> <td><a href="https://www.npmjs.com/">npm</a></td> <td> <ul> @@ -225,7 +233,7 @@ table.supported-languages ul { <td>Y</td> </tr> <tr> - <td>N/A</td> + <td>Not applicable</td> <td><a href="https://classic.yarnpkg.com/en/">yarn</a></td> <td><code>yarn.lock</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -233,7 +241,7 @@ table.supported-languages ul { </tr> <tr> <td>.NET</td> - <td rowspan="2">N/A</td> + <td rowspan="2">Not applicable</td> <td rowspan="2"><a href="https://www.nuget.org/">NuGet</a></td> <td rowspan="2"><a href="https://docs.microsoft.com/en-us/nuget/consume-packages/package-references-in-project-files#enabling-lock-file"><code>packages.lock.json</code></a></td> <td rowspan="2"><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> @@ -267,22 +275,22 @@ table.supported-languages ul { <td> <ul> <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile</code></a></li> - <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile.lock</code></a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-2">2</a></b></sup></li> + <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile.lock</code></a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-3">3</a></b></sup></li> </ul> </td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> - <td><a href="https://python-poetry.org/">Poetry</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-4">4</a></b></sup></td> + <td><a href="https://python-poetry.org/">Poetry</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-5">5</a></b></sup></td> <td><code>poetry.lock</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> <td>Scala</td> - <td>N/A</td> - <td><a href="https://www.scala-sbt.org/">sbt</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-3">3</a></b></sup></td> + <td>Not applicable</td> + <td><a href="https://www.scala-sbt.org/">sbt</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-4">4</a></b></sup></td> <td><code>build.sbt</code></td> <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> @@ -294,13 +302,19 @@ table.supported-languages ul { <li> <a id="notes-regarding-supported-languages-and-package-managers-1"></a> <p> + This version of Java is not supported by the FIPS-enabled image of <code>gemnasium-maven</code>. + </p> + </li> + <li> + <a id="notes-regarding-supported-languages-and-package-managers-2"></a> + <p> Although Gradle with Java 8 is supported, there are other issues such that Android project builds are not supported at this time. Please see the backlog issue <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/336866">Android support for Dependency Scanning (gemnasium-maven)</a> for more details. </p> </li> <li> - <a id="notes-regarding-supported-languages-and-package-managers-2"></a> + <a id="notes-regarding-supported-languages-and-package-managers-3"></a> <p> The presence of a <code>Pipfile.lock</code> file alone will <i>not</i> trigger the analyzer; the presence of a <code>Pipfile</code> is still required in order for the analyzer to be executed. However, if a <code>Pipfile.lock</code> file is found, it will be used by @@ -313,13 +327,13 @@ table.supported-languages ul { </p> </li> <li> - <a id="notes-regarding-supported-languages-and-package-managers-3"></a> + <a id="notes-regarding-supported-languages-and-package-managers-4"></a> <p> Support for <a href="https://www.scala-sbt.org/">sbt</a> 1.3 and above was added in GitLab 13.9. </p> </li> <li> - <a id="notes-regarding-supported-languages-and-package-managers-4"></a> + <a id="notes-regarding-supported-languages-and-package-managers-5"></a> <p> Support for <a href="https://python-poetry.org/">Poetry</a> projects with a <code>poetry.lock</code> file was <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/7006">added in GitLab 15.0</a>. Support for projects without a <code>poetry.lock</code> file is tracked in issue: @@ -342,10 +356,10 @@ The following package managers use lockfiles that GitLab analyzers are capable o | Package Manager | Supported File Format Versions | Tested Versions | | ------ | ------ | ------ | -| Bundler | N/A | [1.17.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/ruby-bundler/default/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | -| Composer | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/php-composer/default/composer.lock) | +| Bundler | Not applicable | [1.17.3](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/ruby-bundler/default/Gemfile.lock#L118), [2.1.4](https://gitlab.com/gitlab-org/security-products/tests/ruby-bundler/-/blob/bundler2-FREEZE/Gemfile.lock#L118) | +| Composer | Not applicable | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/php-composer/default/composer.lock) | | Conan | 0.4 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/c-conan/default/conan.lock) | -| Go | N/A | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/go-modules/default/go.sum) | +| Go | Not applicable | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/go-modules/default/go.sum) | | NuGet | v1 | [4.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/csharp-nuget-dotnetcore/default/src/web.api/packages.lock.json#L2) | | npm | v1, v2 | [6.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/default/package-lock.json#L4), [7.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/lockfileVersion2/package-lock.json#L4) | | yarn | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/default/yarn.lock#L2) | @@ -420,6 +434,8 @@ GitLab relies on [`rules:exists`](../../../ci/yaml/index.md#rulesexists) to star The current detection logic limits the maximum search depth to two levels. For example, the `gemnasium-dependency_scanning` job is enabled if a repository contains either a `Gemfile.lock`, `api/Gemfile.lock`, or `api/client/Gemfile.lock`, but not if the only supported dependency file is `api/v1/client/Gemfile.lock`. +When a supported dependency file is detected, all dependencies, including transitive dependencies are analyzed. There is no limit to the depth of nested or transitive dependencies that are analyzed. + ### How multiple files are processed NOTE: @@ -597,7 +613,7 @@ The following variables are used for configuring specific analyzers (used for a | `GEMNASIUM_DB_REF_NAME` | `gemnasium` | `master` | Branch name for remote repository database. `GEMNASIUM_DB_REMOTE_URL` is required. | | `DS_REMEDIATE` | `gemnasium` | `"true"` | Enable automatic remediation of vulnerable dependencies. | | `GEMNASIUM_LIBRARY_SCAN_ENABLED` | `gemnasium` | `"true"` | Enable detecting vulnerabilities in vendored JavaScript libraries. For now, `gemnasium` leverages [`Retire.js`](https://github.com/RetireJS/retire.js) to do this job. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350512) in GitLab 14.8. | -| `DS_JAVA_VERSION` | `gemnasium-maven` | `17` | Version of Java. Available versions: `8`, `11`, `13`, `14`, `15`, `16`, `17`. | +| `DS_JAVA_VERSION` | `gemnasium-maven` | `17` | Version of Java. Available versions: `8`, `11`, `13`, `14`, `15`, `16`, `17`. Available versions in FIPS-enabled image: `8`, `11`, `17`. | | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that are passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repositories). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that are passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer passes to `sbt`. | @@ -606,6 +622,7 @@ The following variables are used for configuring specific analyzers (used for a | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | +| `DS_INCLUDE_DEV_DEPENDENCIES` | `gemnasium` | `"true"` | When set to `"false"`, development dependencies and their vulnerabilities are not reported. Only NPM projects are supported. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227861) in GitLab 15.1. | #### Other variables @@ -668,6 +685,9 @@ Gemnasium scanning jobs automatically use FIPS-enabled image when FIPS mode is e To manually switch to FIPS-enabled images, set the variable `DS_IMAGE_SUFFIX` to `"-fips"`. +To ensure compliance with FIPS, the FIPS-enabled image of `gemnasium-maven` uses the OpenJDK packages for RedHat UBI. +As a result, it only supports Java 8, 11, and 17. + ## Interacting with the vulnerabilities Once a vulnerability is found, you can interact with it. Read more on how to @@ -1030,7 +1050,7 @@ ensure that it can reach your private repository. Here is an example configurati setuptools.ssl_support.cert_paths = ['internal.crt'] ``` -## Hosting a copy of the gemnasium_db advisory database +## Hosting a copy of the `gemnasium_db` advisory database The [`gemnasium_db`](https://gitlab.com/gitlab-org/security-products/gemnasium-db) Git repository is used by `gemnasium`, `gemnasium-maven`, and `gemnasium-python` as the source of vulnerability data. @@ -1225,7 +1245,7 @@ version `58.1.0+`, which doesn't support `2to3`. Therefore, a `setuptools` depen error in <dependency name> setup command: use_2to3 is invalid ``` -To work around this error, downgrade the analyzer's version of `setuptools` (e.g. `v57.5.0`): +To work around this error, downgrade the analyzer's version of `setuptools` (for example, `v57.5.0`): ```yaml gemnasium-python-dependency_scanning: diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index b5d39f3b32a..d449fbb9a6c 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -12,35 +12,69 @@ GitLab can check your application for security vulnerabilities including: - Data leaks. - Denial of Service (DoS) attacks. +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For an overview of GitLab application security, see [Shifting Security Left](https://www.youtube.com/watch?v=XnYstHObqlA&t). + Statistics and details on vulnerabilities are included in the merge request. Providing actionable information _before_ changes are merged enables you to be proactive. -GitLab also provides high-level statistics of vulnerabilities across projects and groups: +To help with the task of managing and addressing vulnerabilities, GitLab provides a security +dashboard you can access from your project or group. For more details, see +[Security Dashboard](security_dashboard/index.md). -- The [Security Dashboard](security_dashboard/index.md) provides a - high-level view of vulnerabilities detected in your projects, pipeline, and groups. +## Application coverage -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For an overview of GitLab application security, see [Shifting Security Left](https://www.youtube.com/watch?v=XnYstHObqlA&t). +GitLab analyzes various details of your application, either as part of your CI/CD pipeline or on a +schedule. Coverage includes: + +- Source code. +- Dependencies in your projects or container images. +- Vulnerabilities in a running web application. +- Infrastructure as code configuration. + +### Source code analysis + +Source code analysis occurs on every code commit. Details of vulnerabilities detected are provided +in the merge request. + +A source code analysis can: + +- Analyze source code for vulnerabilities - [Static Application Security Testing (SAST)](sast/index.md). +- Analyze the Git repository's history for secrets - [Secret Detection](secret_detection/index.md). + +### Analysis of the running web application + +Analysis of the web application occurs on every code commit. As part of the CI/CD pipeline, your +application is built, deployed to a test environment, and subjected to the following tests: + +- Test for known application vectors - [Dynamic Application Security Testing (DAST)](dast/index.md). +- Analysis of APIs for known attack vectors - [DAST API](dast_api/index.md). +- Analysis of web APIs for unknown bugs and vulnerabilities - [API fuzzing](api_fuzzing/index.md). -## Security scanning tools - -GitLab uses the following tools to scan and report known vulnerabilities found in your project. - -| Secure scanning tool | Description | -| :------------------------------------------------------------- | :------------------------------------------------------------------ | -| [Container Scanning](container_scanning/index.md) | Scan Docker containers for known vulnerabilities. | -| [Dependency List](dependency_list/index.md) | View your project's dependencies and their known vulnerabilities. | -| [Dependency Scanning](dependency_scanning/index.md) | Analyze your dependencies for known vulnerabilities. | -| [Dynamic Application Security Testing (DAST)](dast/index.md) | Analyze running web applications for known vulnerabilities. | -| [DAST API](dast_api/index.md) | Analyze running web APIs for known vulnerabilities. | -| [API fuzzing](api_fuzzing/index.md) | Find unknown bugs and vulnerabilities in web APIs with fuzzing. | -| [Secret Detection](secret_detection/index.md) | Analyze Git history for leaked secrets. | -| [Security Dashboard](security_dashboard/index.md) | View vulnerabilities in all your projects and groups. | -| [Static Application Security Testing (SAST)](sast/index.md) | Analyze source code for known vulnerabilities. | -| [Infrastructure as Code (IaC) Scanning](iac_scanning/index.md) | Analyze your IaC configuration files for known vulnerabilities. | -| [Coverage fuzzing](coverage_fuzzing/index.md) | Find unknown bugs and vulnerabilities with coverage-guided fuzzing. | -| [Cluster Image Scanning](cluster_image_scanning/index.md) | Scan Kubernetes clusters for known vulnerabilities. | +### Dependency analysis + +Dependency analysis occurs on every code commit. Your application's dependencies are collated and +checked against a database of known vulnerabilities. + +Dependency analysis can run: + +- At build time - [Dependency Scanning](dependency_scanning/index.md). +- For projects that use container images, also after the final container + image is built - [Container Scanning](container_scanning/index.md). + +For more details, see +[Dependency Scanning compared to Container Scanning](dependency_scanning/index.md#dependency-scanning-compared-to-container-scanning). + +Additionally, dependencies in operational container images can be analyzed for vulnerabilities +on a regular schedule or cadence. For more details, see [Cluster Image Scanning](cluster_image_scanning/index.md). + +### Infrastructure analysis + +Your application's infrastructure is a source of potential vulnerabilities. To help defend +against this, infrastructure analysis occurs on every merge request. Checks are run against: + +- Infrastructure as Code (IaC) configuration files that define your application's deployment + environment - [Infrastructure as Code (IaC) Scanning](iac_scanning/index.md). ## Vulnerability scanner maintenance @@ -102,15 +136,17 @@ variables: DAST_WEBSITE: https://staging.example.com ``` -For more details about each of the security scanning tools, see their respective -[documentation sections](#security-scanning-tools). - ### Override the default registry base address By default, GitLab security scanners use `registry.gitlab.com/security-products` as the -base address for Docker images. You can override this globally by setting the CI/CD variable +base address for Docker images. You can override this for most scanners by setting the CI/CD variable `SECURE_ANALYZERS_PREFIX` to another location. Note that this affects all scanners at once. +The [Container Scanning](container_scanning/index.md) analyzer is an exception, and it +does not use the `SECURE_ANALYZERS_PREFIX` variable. To override its Docker image, see +the instructions for [Running container scanning in an offline +environment](container_scanning/index.md#running-container-scanning-in-an-offline-environment). + ### Use security scanning tools with merge request pipelines By default, the application security jobs are configured to run for branch pipelines only. @@ -126,19 +162,19 @@ rules: ### Secure jobs in your pipeline -If you add the security scanning jobs as described in [Security scanning with Auto DevOps](#security-scanning-with-auto-devops) or [Security scanning without Auto DevOps](#security-scanning-without-auto-devops) to your `.gitlab-ci.yml` each added [security scanning tool](#security-scanning-tools) behave as described below. +If you add the security scanning jobs as described in [Security scanning with Auto DevOps](#security-scanning-with-auto-devops) or [Security scanning without Auto DevOps](#security-scanning-without-auto-devops) to your `.gitlab-ci.yml` each added [security scanning tool](#application-coverage) behave as described below. For each compatible analyzer, a job is created in the `test`, `dast` or `fuzz` stage of your pipeline and runs on the next new branch pipeline. Features such as the [Security Dashboard](security_dashboard/index.md), [Vulnerability Report](vulnerability_report/index.md), and [Dependency List](dependency_list/index.md) that rely on this scan data only show results from pipelines on the default branch, only if all jobs are finished, including manual ones. One tool might use many analyzers. -Our language and package manager specific jobs attempt to assess which analyzer(s) they should run for your project so that you can do less configuration. +Our language and package manager specific jobs attempt to assess which analyzers they should run for your project so that you can do less configuration. If you want to override this to increase the pipeline speed you may choose which analyzers to exclude if you know they are not applicable (languages or package managers not contained in your project) by following variable customization directions for that specific tool. ### Secure job status -Jobs pass if they are able to complete a scan. A _pass_ result does NOT indicate if they did, or did not, identify findings. The only exception is coverage fuzzing, which fails if it identifies findings. +Jobs pass if they are able to complete a scan. A _pass_ result does not indicate if they did, or did not, identify findings. The only exception is coverage fuzzing, which fails if it identifies findings. Jobs fail if they are unable to complete a scan. You can view the pipeline logs for more information. @@ -169,11 +205,11 @@ reports are available to download. To download a report, select ### Ultimate -A merge request contains a security widget which displays a summary of the NEW results. New results are determined by comparing the current findings against existing findings in the target (default) branch (if there are prior findings). +A merge request contains a security widget which displays a summary of the new results. New results are determined by comparing the current findings against existing findings in the target (default) branch (if there are prior findings). We recommend you run a scan of the `default` branch before enabling feature branch scans for your developers. Otherwise, there is no base for comparison and all feature branches display the full scan results in the merge request security widget. -The merge request security widget displays only a subset of the vulnerabilities in the generated JSON artifact because it contains both NEW and EXISTING findings. +The merge request security widget displays only a subset of the vulnerabilities in the generated JSON artifact because it contains both new and existing findings. From the merge request security widget, select **Expand** to unfold the widget, displaying any new and no longer detected (removed) findings by scan type. Select **View full report** to go directly to the **Security** tab in the latest branch pipeline. @@ -213,9 +249,9 @@ security issues: ### Migration of existing Vulnerability-Check rules -If your projects have rules that have a security orchestration project, a new MR with +If your projects have rules that have a security orchestration project, a new MR with the existing rule's content is created automatically against the default branch belonging -to the security orchestration project. To maintain the same security approval rules you +to the security orchestration project. To maintain the same security approval rules you had before GitLab 15.0, we recommend merging this new MR. If your projects have rules without a security orchestration project, a new security orchestration project is created automatically with the content of the existing rule. No additional action is required. @@ -360,10 +396,17 @@ Self managed installations can also run the security scanners on a GitLab Runner You can enforce validation of the security report artifacts before ingesting the vulnerabilities. This prevents ingestion of broken vulnerability data into the database. GitLab validates the artifacts based on the [report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tree/master/dist). - -In GitLab 14.0 and later, when artifact validation is enabled, the pipeline's **Security** tab lists +When artifact validation is enabled, the pipeline's **Security** tab lists any report artifacts that failed validation. +Validation depends on the schema: + +- If your security report does not specify which schema version it uses, GitLab attempts to verify it against the earliest supported schema version for that report type. Validation fails but it's attempted anyway because it may identify other problems present in the report. +- If your security report uses a version that is not supported, GitLab attempts to validate it against the earliest supported schema version for that report type. Validation fails but will identify the differences between the schema version used and the earliest supported version. +- If your security report uses a deprecated version, GitLab attempts validation against that version and adds a warning to the validation result. + +You can always find supported and deprecated schema versions in the [source code](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/parsers/security/validators/schema_validator.rb#L9). + ### Enable security report validation > [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/354928) in GitLab 14.9, and planned for removal in GitLab 15.0. @@ -553,18 +596,18 @@ When overriding the template to control job execution, previous instances of [`only` or `except`](../../ci/yaml/index.md#only--except) are no longer compatible and must be transitioned to [the `rules` syntax](../../ci/yaml/index.md#rules). -If your override is aimed at limiting jobs to only run on `master`, the previous syntax +If your override is aimed at limiting jobs to only run on `main`, the previous syntax would look similar to: ```yaml include: - template: Security/SAST.gitlab-ci.yml -# Ensure that the scanning is only executed on master or merge requests +# Ensure that the scanning is only executed on main or merge requests spotbugs-sast: only: refs: - - master + - main - merge_requests ``` @@ -575,10 +618,10 @@ would be written as follows: include: - template: Security/SAST.gitlab-ci.yml -# Ensure that the scanning is only executed on master or merge requests +# Ensure that the scanning is only executed on main or merge requests spotbugs-sast: rules: - - if: $CI_COMMIT_BRANCH == "master" + - if: $CI_COMMIT_BRANCH == "main" - if: $CI_MERGE_REQUEST_ID ``` @@ -659,6 +702,3 @@ These security pages can be populated by running the jobs from the manual step o There is [an issue open to handle this scenario](https://gitlab.com/gitlab-org/gitlab/-/issues/346843). Please upvote the issue to help with prioritization, and [contributions are welcomed](https://about.gitlab.com/community/contribute/). - doc/user/project/merge_requests/approvals/settings.md -+ -0 diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index f790164d9a0..27a6f867ae2 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -121,6 +121,10 @@ Additionally, if you are a project owner and a security policy project has not b associated with this project, then a new project is created and associated automatically at the same time that the first policy merge request is created. +## Managing projects in bulk via a script + +You can use the [Vulnerability-Check Migration](https://gitlab.com/gitlab-org/gitlab/-/snippets/2328089) script to bulk create policies or associate security policy projects with development projects. For instructions and a demonstration of how to use the Vulnerability-Check Migration script, see [this video](https://youtu.be/biU1N26DfBc). + ## Scan execution policies See [Scan execution policies](scan-execution-policies.md). diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md index aa23ad30a73..5beb6912877 100644 --- a/doc/user/application_security/policies/scan-execution-policies.md +++ b/doc/user/application_security/policies/scan-execution-policies.md @@ -47,7 +47,8 @@ The policy editor currently only supports the YAML mode. The Rule mode is tracke The YAML file with scan execution policies consists of an array of objects matching scan execution policy schema nested under the `scan_execution_policy` key. You can configure a maximum of 5 -policies under the `scan_execution_policy` key. +policies under the `scan_execution_policy` key. Any other policies configured after +the first 5 are not applied. When you save a new policy, GitLab validates its contents against [this JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/validators/json_schemas/security_orchestration_policy.json). If you're not familiar with how to read [JSON schemas](https://json-schema.org/), @@ -85,9 +86,8 @@ This rule enforces the defined actions and schedules a scan on the provided date | `type` | `string` | `schedule` | The rule's type. | | `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). | | `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. | -| `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [cluster image scanning](../../clusters/agent/vulnerabilities.md) will run. The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. | <!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> -| `clusters` (removed) | `object` | | This field was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in 15.0. Use the `agents` field instead. The cluster where the given policy enforces running selected scans (only for `container_scanning`/`cluster_image_scanning` scans). The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. | -<!--- end_remove --> +| `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [cluster image scanning](../../clusters/agent/vulnerabilities.md) will run. The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. <!--- start_remove The following content will be removed on remove_date: '2022-08-22' --> | +| `clusters` (removed) | `object` | | This field was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/356465) in 15.0. Use the `agents` field instead. The cluster where the given policy enforces running selected scans (only for `container_scanning`/`cluster_image_scanning` scans). The key of the object is the name of the Kubernetes cluster configured for your project in GitLab. In the optionally provided value of the object, you can precisely select Kubernetes resources that are scanned. <!--- end_remove --> | GitLab supports the following types of CRON syntax for the `cadence` field: @@ -152,7 +152,7 @@ Note the following: mode when executed as part of a scheduled scan. - A container scanning and cluster image scanning scans configured for the `pipeline` rule type ignores the cluster defined in the `clusters` object. They use predefined CI/CD variables defined for your project. Cluster selection with the `clusters` object is supported for the `schedule` rule type. - Cluster with name provided in `clusters` object must be created and configured for the project. To be able to successfully perform the `container_scanning`/`cluster_image_scanning` scans for the cluster you must follow instructions for the [Cluster Image Scanning feature](../cluster_image_scanning/index.md#prerequisites). + A cluster with a name provided in the `clusters` object must be created and configured for the project. - The SAST scan uses the default template and runs in a [child pipeline](../../../ci/pipelines/parent_child_pipelines.md). ## Example security policies project diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index 232a5c9f91c..3da884aca6a 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -91,7 +91,7 @@ the defined policy. Requirements and limitations: -- You must add the respective [security scanning tools](../index.md#security-scanning-tools). +- You must add the respective [security scanning tools](../index.md#application-coverage). Otherwise, scan result policies won't have any effect. - The maximum number of policies is five. - Each policy can have a maximum of five rules. diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 661f564828a..0f9e3343072 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -63,35 +63,6 @@ content directly. Instead, it enhances the results with additional properties, i - Location tracking fields. - A means of identifying false positives or insignificant findings. **(ULTIMATE)** -## Data provided by analyzers - -Each analyzer provides data about the vulnerabilities it detects. The following table details the -data available from each analyzer. The values provided by these tools are heterogeneous so they are sometimes -normalized into common values, for example, `severity` and `confidence`. - -| Property / tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Semgrep | Sobelow | -|--------------------------------|------|--------|----------|-----------------|----------|------------|-------|-----------------|-------|------------|-----------------------|---------------------------|---------|---------| -| Affected item (for example, class or package) | ✓ | ✗ | ✓ | ✗ | ✓ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| Confidence | ✗ | ✓ | ✓ | ✗ | ✓ | x | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✓ | -| Description | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | -| End column | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| End line | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| External ID (for example, CVE) | ✗ | ✗ | ⚠ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | -| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Internal doc/explanation | ✓ | ⚠ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | -| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | -| Severity | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ⚠ | ✗ | -| Solution | ✓ | ✗ | ✗ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | -| Source code extract | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | -| Start column | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | -| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| URLs | ✓ | ✗ | ✓ | ✗ | ⚠ | ✗ | ⚠ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | - -- ✓ => Data is available. -- ⚠ => Data is available, but it's partially reliable, or it has to be extracted from unstructured content. -- ✗ => Data is not available or it would require specific, inefficient or unreliable, logic to obtain it. - ## Transition to Semgrep-based scanning SAST includes a [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) that covers [multiple languages](index.md#supported-languages-and-frameworks). @@ -120,10 +91,10 @@ The Vulnerability Management system automatically moves vulnerabilities from the However, you'll see old vulnerabilities re-created based on Semgrep results if: -- A vulnerability was created by Bandit or SpotBugs and you disable those analyzers. We only recommend disabling Bandit and SpotBugs now if the analyzers aren’t working. Work to automatically translate Bandit and SpotBugs vulnerabilities to Semgrep is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328062). +- A vulnerability was created by Bandit or SpotBugs and you disable those analyzers. We only recommend disabling Bandit and SpotBugs now if the analyzers aren't working. Work to automatically translate Bandit and SpotBugs vulnerabilities to Semgrep is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328062). - A vulnerability was created by ESLint, Gosec, or Flawfinder in a default-branch pipeline where Semgrep scanning did not run successfully (before Semgrep coverage was introduced for the language, because you disabled Semgrep explicitly, or because the Semgrep scan failed in that pipeline). We do not currently plan to combine these vulnerabilities if they already exist. -When a vulnerability is re-created, the original vulnerability is marked as “no longer detected” in the Vulnerability Report. +When a vulnerability is re-created, the original vulnerability is marked as "no longer detected" in the Vulnerability Report. A new vulnerability is then created based on the Semgrep finding. ### Activating Semgrep-based scanning early @@ -184,7 +155,7 @@ variables: You can disable all default SAST analyzers, leaving only [custom analyzers](#custom-analyzers) enabled. -To disable all default analyzers, set the CI/CD variable `SAST_DISABLED` to `true` in your +To disable all default analyzers, set the CI/CD variable `SAST_DISABLED` to `"true"` in your `.gitlab-ci.yml` file. Example: @@ -194,7 +165,7 @@ include: - template: Security/SAST.gitlab-ci.yml variables: - SAST_DISABLED: true + SAST_DISABLED: "true" ``` ### Disable specific default analyzers @@ -241,3 +212,32 @@ csharp-sast: reports: sast: gl-sast-report.json ``` + +## Data provided by analyzers + +Each analyzer provides data about the vulnerabilities it detects. The following table details the +data available from each analyzer. The values provided by these tools are heterogeneous so they are sometimes +normalized into common values, for example, `severity` and `confidence`. + +| Property / tool | Apex | Bandit | Brakeman | ESLint security | SpotBugs | Flawfinder | Gosec | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Semgrep | Sobelow | +|--------------------------------|------|--------|----------|-----------------|----------|------------|-------|-----------------|-------|------------|-----------------------|---------------------------|---------|---------| +| Affected item (for example, class or package) | ✓ | ✗ | ✓ | ✗ | ✓ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Confidence | ✗ | ✓ | ✓ | ✗ | ✓ | x | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✓ | +| Description | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | ✓ | +| End column | ✓ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| End line | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| External ID (for example, CVE) | ✗ | ✗ | ⚠ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | +| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Internal doc/explanation | ✓ | ⚠ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | +| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | +| Severity | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ⚠ | ✗ | +| Solution | ✓ | ✗ | ✗ | ✗ | ⚠ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✗ | +| Source code extract | ✗ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Start column | ✓ | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ✗ | +| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| URLs | ✓ | ✗ | ✓ | ✗ | ⚠ | ✗ | ⚠ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | + +- ✓ => Data is available. +- ⚠ => Data is available, but it's partially reliable, or it has to be extracted from unstructured content. +- ✗ => Data is not available or it would require specific, inefficient or unreliable, logic to obtain it. diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 38f26b7578d..d4dd8059c6a 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -13,7 +13,7 @@ The whitepaper ["A Seismic Shift in Application Security"](https://about.gitlab. explains how 4 of the top 6 attacks were application based. Download it to learn how to protect your organization. -If you’re using [GitLab CI/CD](../../../ci/index.md), you can use Static Application Security +If you're using [GitLab CI/CD](../../../ci/index.md), you can use Static Application Security Testing (SAST) to check your source code for known vulnerabilities. You can run SAST analyzers in any GitLab tier. The analyzers output JSON-formatted reports as job artifacts. @@ -182,6 +182,7 @@ as shown in the following table: | [Configure SAST in the UI](#configure-sast-in-the-ui) | **{dotted-circle}** | **{check-circle}** | | [Customize SAST rulesets](#customize-rulesets) | **{dotted-circle}** | **{check-circle}** | | [Detect False Positives](#false-positive-detection) | **{dotted-circle}** | **{check-circle}** | +| [Track moved vulnerabilities](#advanced-vulnerability-tracking) | **{dotted-circle}** | **{check-circle}** | ## Contribute your scanner @@ -524,7 +525,7 @@ defined for the `nodejs-scan` scanner: ''' ``` -##### File passthrough for gosec +##### File passthrough for Gosec Provide the name of the file containing a custom analyzer configuration. In this example, customized rules for the `gosec` scanner are contained in the @@ -539,7 +540,7 @@ file `gosec-config.json`: value = "gosec-config.json" ``` -##### Passthrough chain for semgrep +##### Passthrough chain for Semgrep In the below example, we generate a custom configuration under the `/sgrules` target directory with a total `timeout` of 60 seconds. @@ -560,7 +561,7 @@ Several passthrouh types generate a configuration for the target analyzer: - The `url` entry fetches a configuration made available through a URL and stores it in the `/sgrules/gosec.yml` file. -Afterwards, semgrep is invoked with the final configuration located under +Afterwards, Semgrep is invoked with the final configuration located under `/sgrules`. ```toml @@ -632,12 +633,12 @@ created when preceding passthroughs in the chain find a naming collision. If `mode` is set to `append`, a passthrough appends data to the files created by its predecessors instead of overwriting. -In the below semgrep configuration,`/sgrules/insecure.yml` assembles two passthroughs. The rules are: +In the below Semgrep configuration,`/sgrules/insecure.yml` assembles two passthroughs. The rules are: - `insecure` - `secret` -These rules add a search pattern to the analyzer and extends semgrep capabilities. +These rules add a search pattern to the analyzer and extends Semgrep capabilities. For passthrough chains we recommend that you enable validation. To enable validation, you can either: @@ -696,6 +697,32 @@ rules: Vulnerabilities that have been detected and are false positives will be flagged as false positives in the security dashboard. +False positive detection is available in a subset of the [supported languages](#supported-languages-and-frameworks) and [analyzers](analyzers.md): + +- Ruby, in the Brakeman-based analyzer + +### Advanced vulnerability tracking **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5144) in GitLab 14.2. + +Source code is volatile; as developers make changes, source code may move within files or between files. +Security analyzers may have already reported vulnerabilities that are being tracked in the [Vulnerability Report](../vulnerability_report/). +These vulnerabilities are linked to specific problematic code fragments so that they can be found and fixed. +If the code fragments are not tracked reliably as they move, vulnerability management is harder because the same vulnerability could be reported again. + +GitLab SAST uses an advanced vulnerability tracking algorithm to more accurately identify when the same vulnerability has moved within a file due to refactoring or unrelated changes. + +Advanced vulnerability tracking is available in a subset of the [supported languages](#supported-languages-and-frameworks) and [analyzers](analyzers.md): + +- C, in the Semgrep-based analyzer only +- Go, in the Gosec- and Semgrep-based analyzers +- Java, in the Semgrep-based analyzer only +- JavaScript, in the Semgrep-based analyzer only +- Python, in the Semgrep-based analyzer only +- Ruby, in the Brakeman-based analyzer + +Support for more languages and analyzers is tracked in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/5144). + ### Using CI/CD variables to pass credentials for private repositories Some analyzers require downloading the project's dependencies in order to diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 3937cbd77b6..9805fb3b67c 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -197,7 +197,8 @@ Secret Detection can be customized by defining available CI/CD variables: |-----------------------------------|---------------|-------------| | `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | -| `SECRET_DETECTION_IMAGE_SUFFIX` | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. | +| `SECRET_DETECTION_IMAGE_SUFFIX` | "" | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. | +| `SECRET_DETECTION_LOG_OPTIONS` | "" | [`git log`](https://git-scm.com/docs/git-log) options used to define commit ranges. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350660) in GitLab 15.1.| In previous GitLab versions, the following variables were also available: @@ -220,7 +221,7 @@ simultaneously: - [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). - [Overriding predefined rules](index.md#override-predefined-analyzer-rules). -- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. +- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Customization allows replacing the default secret detection rules with rules that you define. diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 577606885ca..3cb4bd4a02d 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -7,13 +7,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Security Dashboards and Security Center **(ULTIMATE)** -You can use Security Dashboards to view details about vulnerabilities -detected by [security scanners](../index.md#security-scanning-tools). -These details are shown in pipelines, projects, and groups. +You can use Security Dashboards to view trends about vulnerabilities +detected by [security scanners](../index.md#application-coverage). +These trends are shown in projects, groups, and the Security Center. To use the Security Dashboards, you must: -- Configure at least one [security scanner](../index.md#security-scanning-tools) in a project. +- Configure at least one [security scanner](../index.md#application-coverage) in a project. - Configure jobs to use the [`reports` syntax](../../../ci/yaml/index.md#artifactsreports). - Use [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or later. If you use the shared runners on GitLab.com, you are using the correct version. diff --git a/doc/user/application_security/terminology/index.md b/doc/user/application_security/terminology/index.md index 8277c30b81f..392bfa1dde2 100644 --- a/doc/user/application_security/terminology/index.md +++ b/doc/user/application_security/terminology/index.md @@ -96,6 +96,106 @@ A finding's location fingerprint is a text value that's unique for each location surface. Each Secure product defines this according to its type of attack surface. For example, SAST incorporates file path and line number. +### Package managers + +A Package manager is a system that manages your project dependencies. + +The package manager provides a method to install new dependencies (also referred to as "packages"), manage where packages are stored on your file system, and offer capabilities for you to publish your own packages. + +### Package types + +Each package manager, platform, type, or ecosystem has its own conventions and protocols to identify, locate, and provision software packages. + +The following table is a non-exhaustive list of some of the package managers and types referenced in GitLab documentation and software tools. + +<style> +table.package-managers-and-types tr:nth-child(even) { + background-color: transparent; +} + +table.package-managers-and-types td { + border-left: 1px solid #dbdbdb; + border-right: 1px solid #dbdbdb; + border-bottom: 1px solid #dbdbdb; +} + +table.package-managers-and-types tr td:first-child { + border-left: 0; +} + +table.package-managers-and-types tr td:last-child { + border-right: 0; +} + +table.package-managers-and-types ul { + font-size: 1em; + list-style-type: none; + padding-left: 0px; + margin-bottom: 0px; +} +</style> + +<table class="package-managers-and-types"> + <thead> + <tr> + <th>Package Type</th> + <th>Package Manager</th> + </tr> + </thead> + <tbody> + <tr> + <td>gem</td> + <td><a href="https://bundler.io/">bundler</a></td> + </tr> + <tr> + <td>packagist</td> + <td><a href="https://getcomposer.org/">composer</a></td> + </tr> + <tr> + <td>conan</td> + <td><a href="https://conan.io/">conan</a></td> + </tr> + <tr> + <td>go</td> + <td><a href="https://go.dev/blog/using-go-modules">go</a></td> + </tr> + <tr> + <td rowspan="3">maven</td> + <td><a href="https://gradle.org/">gradle</a></td> + </tr> + <tr> + <td><a href="https://maven.apache.org/">maven</a></td> + </tr> + <tr> + <td><a href="https://www.scala-sbt.org">sbt</a></td> + </tr> + <tr> + <td rowspan="2">npm</td> + <td><a href="https://www.npmjs.com">npm</a></td> + </tr> + <tr> + <td><a href="https://classic.yarnpkg.com/en">yarn</a></td> + </tr> + <tr> + <td>nuget</td> + <td><a href="https://www.nuget.org/">nuget</a></td> + </tr> + <tr> + <td rowspan="4">pypi</td> + <td><a href="https://setuptools.pypa.io/en/latest/">setuptools</a></td> + </tr> + <tr> + <td><a href="https://pip.pypa.io/en/stable">pip</a></td> + </tr> + <tr> + <td><a href="https://pipenv.pypa.io/en/latest">Pipenv</a></td> + </tr> + <tr> + <td><a href="https://python-poetry.org/">Poetry</a></td> + </tr> + </tbody> +</table> + ### Pipeline Security tab A page that displays findings discovered in the associated CI pipeline. diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index 87344e4ff65..f5b1192269d 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -4,13 +4,14 @@ group: Threat Insights info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Vulnerability Pages **(ULTIMATE)** +# Vulnerability Page **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13561) in GitLab 13.0. -Each vulnerability in a project has a Vulnerability Page. This page contains details of the -vulnerability. The details included vary according to the type of vulnerability. Details of each -vulnerability include: +Each vulnerability in a project has a Vulnerability Page, containing details of the +vulnerability. The details included vary according to the type of vulnerability. + +Details of each vulnerability include: - Description - When it was detected @@ -24,32 +25,41 @@ alert message is included at the top of the vulnerability's page. On the vulnerability's page, you can: -- [Change the vulnerability's status](#change-vulnerability-status). -- [Create an issue](#create-an-issue-for-a-vulnerability). -- [Link issues to the vulnerability](#linked-issues). -- [Resolve a vulnerability](#resolve-a-vulnerability) if a solution is +- [Change the vulnerability's status](#change-status-of-a-vulnerability). +- [Create an issue](#creating-an-issue-for-a-vulnerability). +- [Link issues to the vulnerability](#linking-a-vulnerability-to-issues). +- [Resolve the vulnerability](#resolve-a-vulnerability) if a solution is available. - [View security training specific to the detected vulnerability](#view-security-training-for-a-vulnerability). ## Vulnerability status values -A vulnerability's status can be one of the following: +A vulnerability's status can be: + +- **Detected**: The default state for a newly discovered vulnerability. Appears as "Needs triage" in the UI. +- **Confirmed**: A user has seen this vulnerability and confirmed it to be accurate. +- **Dismissed**: A user has seen this vulnerability and dismissed it because it is not accurate or otherwise not to be resolved. +- **Resolved**: The vulnerability has been fixed or is no longer present. -| Status | Description | -|:----------|:------------| -| Detected | The default state for a newly discovered vulnerability. Appears as "Needs triage" in the UI. | -| Confirmed | A user has seen this vulnerability and confirmed it to be accurate. | -| Dismissed | A user has seen this vulnerability and dismissed it because it is not accurate or otherwise not to be resolved. | -| Resolved | The vulnerability has been fixed or is no longer present. | +Dismissed vulnerabilities are ignored if detected in subsequent scans. Resolved vulnerabilities that +are reintroduced and detected by subsequent scans have a _new_ vulnerability record created. When an +existing vulnerability is no longer detected in a project's `default` branch, you should change its +status to **Resolved**. This ensures that if it is accidentally reintroduced in a future merge, it +is reported again as a new record. You can use the Vulnerability Report's +[Activity filter](../vulnerability_report/#activity-filter) to select all vulnerabilities that are +no longer detected, and change their status. -Dismissed vulnerabilities are ignored if detected in subsequent scans. Resolved vulnerabilities that are reintroduced and detected by subsequent scans have a _new_ vulnerability record created. When an existing vulnerability is no longer detected in a project's `default` branch, you should change its status to Resolved. This ensures that if it is accidentally reintroduced in a future merge, it will be visible again as a new record. You can use the [Activity filter](../vulnerability_report/#activity-filter) to select all vulnerabilities that are no longer detected, and [change their status](../vulnerability_report#change-status-of-vulnerabilities). +## Change status of a vulnerability -## Change vulnerability status +To change a vulnerability's status from its Vulnerability Page: -To change a vulnerability's status, select a new value from the **Status** dropdown then select -**Change status**. Optionally, add a comment to the log entry at the bottom of the page. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. +1. From the **Status** dropdown list select a status, then select **Change status**. +1. Optionally, at the bottom of the page, add a comment to the log entry. -## Create an issue for a vulnerability +## Creating an issue for a vulnerability From a vulnerability's page you can create an issue to track all action taken to resolve or mitigate it. @@ -67,7 +77,9 @@ that when Jira integration is enabled, the GitLab issue feature is not available To create a GitLab issue for a vulnerability: -1. In GitLab, go to the vulnerability's page. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. 1. Select **Create issue**. An issue is created in the project, pre-populated with information from the vulnerability report. @@ -80,73 +92,85 @@ The issue is then opened so you can take further action. Prerequisites: -- [Enable Jira integration](../../../integration/jira/index.md). - The **Enable Jira issue creation from vulnerabilities** option must be selected as part of the configuration. -- Each user must have a personal Jira user account with permission to create issues in the target project. +- [Enable Jira integration](../../../integration/jira/index.md). The **Enable Jira issue creation + from vulnerabilities** option must be selected as part of the configuration. +- Each user must have a personal Jira user account with permission to create issues in the target + project. To create a Jira issue for a vulnerability: -1. Go to the vulnerability's page. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. 1. Select **Create Jira issue**. 1. If you're not already logged in to Jira, log in. The Jira issue is created and opened in a new browser tab. The **Summary** and **Description** fields are pre-populated from the vulnerability's details. -Unlike GitLab issues, the status of whether a Jira issue is open or closed does not display in the GitLab user interface. +Unlike GitLab issues, the status of whether a Jira issue is open or closed does not display in the +GitLab user interface. -## Linked issues +## Linking a vulnerability to issues NOTE: If Jira issue support is enabled, GitLab issues are disabled so this feature is not available. -You can link one or more existing GitLab issues to a vulnerability. Adding a link helps track +You can link a vulnerability to one or more existing GitLab issues. Adding a link helps track the issue that resolves or mitigates a vulnerability. Issues linked to a vulnerability are shown in the Vulnerability Report and the vulnerability's page. Be aware of the following conditions between a vulnerability and a linked issue: -- The vulnerability page shows related issues, but the issue page doesn't show the vulnerability it's related to. +- The vulnerability page shows related issues, but the issue page doesn't show the vulnerability + it's related to. - An issue can only be related to one vulnerability at a time. - Issues can be linked across groups and projects. -## Link to existing issues +## Link a vulnerability to existing issues To link a vulnerability to existing issues: -1. Go to the vulnerability's page. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. 1. In the **Linked issues** section, select the plus icon (**{plus}**). 1. For each issue to be linked, either: - Paste a link to the issue. - Enter the issue's ID (prefixed with a hash `#`). 1. Select **Add**. -The selected issues are added to the **Linked issues** section, and the linked issues counter is updated. +The selected issues are added to the **Linked issues** section, and the linked issues counter is +updated. ## Resolve a vulnerability For some vulnerabilities a solution is already known. In those instances, a vulnerability's page includes a **Resolve with merge request** option. -To resolve a vulnerability, you can either: - -- [Resolve a vulnerability with a merge request](#resolve-a-vulnerability-with-a-merge-request). -- [Resolve a vulnerability manually](#resolve-a-vulnerability-manually). - -The following scanners are supported: +The following scanners are supported by this feature: - [Dependency Scanning](../dependency_scanning/index.md). Automatic Patch creation is only available for Node.js projects managed with `yarn`. - [Container Scanning](../container_scanning/index.md). +To resolve a vulnerability, you can either: + +- [Resolve a vulnerability with a merge request](#resolve-a-vulnerability-with-a-merge-request). +- [Resolve a vulnerability manually](#resolve-a-vulnerability-manually). +  ### Resolve a vulnerability with a merge request -To resolve the vulnerability with a merge request, go to the vulnerability's page and from the -**Resolve with merge request** dropdown select **Resolve with merge request**. +To resolve the vulnerability with a merge request: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. +1. From the **Resolve with merge request** dropdown list, select **Resolve with merge request**. A merge request is created which applies the patch required to resolve the vulnerability. Process the merge request according to your standard workflow. @@ -155,11 +179,15 @@ Process the merge request according to your standard workflow. To manually apply the patch that GitLab generated for a vulnerability: -1. Go to the vulnerability's page and from the **Resolve with merge request** dropdown select - **Download patch to resolve**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Security & Compliance > Vulnerability report**. +1. Select the vulnerability's description. +1. From the **Resolve with merge request** dropdown list, select **Download patch to resolve**. 1. Ensure your local project has the same commit checked out that was used to generate the patch. 1. Run `git apply remediation.patch`. 1. Verify and commit the changes to your branch. +1. Create a merge request to apply the changes to your main branch. +1. Process the merge request according to your standard workflow. ## Enable security training for vulnerabilities diff --git a/doc/user/application_security/vulnerabilities/severities.md b/doc/user/application_security/vulnerabilities/severities.md index 967a6d9fa89..987dac677e7 100644 --- a/doc/user/application_security/vulnerabilities/severities.md +++ b/doc/user/application_security/vulnerabilities/severities.md @@ -39,17 +39,17 @@ the following tables: |----------------------------------------------------------------------------------------------------------|--------------------------|----------------------------|------------------------------------| | [`security-code-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) | **{check-circle}** Yes | String | `CRITICAL`, `HIGH`, `MEDIUM` in [analyzer version 3.2.0 and later](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan/-/blob/master/CHANGELOG.md#v320). In earlier versions, hardcoded to `Unknown`. | | [`brakeman`](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) | **{check-circle}** Yes | String | `HIGH`, `MEDIUM`, `LOW` | -| [`sobelow`](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) | **{check-circle}** Yes | N/A | Hardcodes all severity levels to `Unknown` | +| [`sobelow`](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) | **{check-circle}** Yes | Not applicable | Hardcodes all severity levels to `Unknown` | | [`nodejs-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) | **{check-circle}** Yes | String | `INFO`, `WARNING`, `ERROR` | | [`flawfinder`](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder) | **{check-circle}** Yes | Integer | `0`, `1`, `2`, `3`, `4`, `5` | -| [`eslint`](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | **{check-circle}** Yes | N/A | Hardcodes all severity levels to `Unknown` | +| [`eslint`](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | **{check-circle}** Yes | Not applicable | Hardcodes all severity levels to `Unknown` | | [`SpotBugs`](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) | **{check-circle}** Yes | Integer | `1`, `2`, `3`, `11`, `12`, `18` | | [`gosec`](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) | **{check-circle}** Yes | String | `HIGH`, `MEDIUM`, `LOW` | | [`bandit`](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) | **{check-circle}** Yes | String | `HIGH`, `MEDIUM`, `LOW` | | [`phpcs-security-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) | **{check-circle}** Yes | String | `ERROR`, `WARNING` | | [`pmd-apex`](https://gitlab.com/gitlab-org/security-products/analyzers/pmd-apex) | **{check-circle}** Yes | Integer | `1`, `2`, `3`, `4`, `5` | | [`kubesec`](https://gitlab.com/gitlab-org/security-products/analyzers/kubesec) | **{check-circle}** Yes | String | `CriticalSeverity`, `InfoSeverity` | -| [`secrets`](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) | **{check-circle}** Yes | N/A | Hardcodes all severity levels to `Critical` | +| [`secrets`](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) | **{check-circle}** Yes | Not applicable | Hardcodes all severity levels to `Critical` | | [`semgrep`](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) | **{check-circle}** Yes | String | `error`, `warning`, `note`, `none` | ## Dependency Scanning diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index e499ddbbd6b..15a287356f8 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -23,7 +23,7 @@ The **Activity** column contains icons to indicate the activity, if any, taken o in that row: - Issues **{issues}**: Links to issues created for the vulnerability. For more details, read - [Create an issue for a vulnerability](../vulnerabilities/index.md#create-an-issue-for-a-vulnerability). + [Create an issue for a vulnerability](../vulnerabilities/index.md#creating-an-issue-for-a-vulnerability). - Wrench **{admin}**: The vulnerability has been remediated. - False positive **{false-positive}**: The scanner determined this vulnerability to be a false positive. @@ -105,7 +105,7 @@ When using the tool filter, you can choose: - Individual GitLab-provided tools. - Any integrated third-party tool. -For details of each of the available tools, see [Security scanning tools](../index.md#security-scanning-tools). +For details of each of the available tools, see [Security scanning tools](../index.md#application-coverage). ### Project filter @@ -249,7 +249,7 @@ To add a new vulnerability finding from your project level Vulnerability Report 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Security & Compliance > Vulnerability Report**. -1. Click on **Submit Vulnerability**. +1. Select **Submit Vulnerability**. 1. Complete the fields and submit the form. You will be brought to the newly created vulnerability's detail page. Manually created records appear in the @@ -259,7 +259,7 @@ Group, Project, and Security Center Vulnerability Reports. To filter them, use t > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6345) in GitLab 14.6. -The **Operational vulnerabilities** tab lists vulnerabilities found by the `cluster_image_scanner`. +The **Operational vulnerabilities** tab lists vulnerabilities found by [Operational container scanning](../../clusters/agent/vulnerabilities.md). This tab appears on the project, group, and Security Center vulnerability reports.  diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index 80c85358fcb..b55a55eebe5 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -235,7 +235,7 @@ v1.0, 2019-01-01 NOTE: [Wiki pages](project/wiki/index.md#create-a-new-wiki-page) created with the AsciiDoc format are saved with the file extension `.asciidoc`. When working with AsciiDoc wiki -pages, change the file name from `.adoc` to `.asciidoc`. +pages, change the filename from `.adoc` to `.asciidoc`. ```plaintext include::basics.adoc[] @@ -381,9 +381,9 @@ comment - content which is not included in the output document It's possible to have color written in `HEX`, `RGB`, or `HSL` format rendered with a color indicator. Supported formats (named colors are not supported): -- HEX: `` `#RGB[A]` `` or `` `#RRGGBB[AA]` `` -- RGB: `` `RGB[A](R, G, B[, A])` `` -- HSL: `` `HSL[A](H, S, L[, A])` `` +- `HEX`: `` `#RGB[A]` `` or `` `#RRGGBB[AA]` `` +- `RGB`: `` `RGB[A](R, G, B[, A])` `` +- `HSL`: `` `HSL[A](H, S, L[, A])` `` Color written inside backticks is followed by a color "chip": @@ -399,10 +399,10 @@ Color written inside backticks is followed by a color "chip": - `HSLA(540,70%,50%,0.3)` ``` -### STEM +### Equations and Formulas (STEM) -To activate equation and formula support, -set the `stem` attribute in the document's header to `latexmath`. +If you need to include Science, Technology, Engineering and Math (STEM) +expressions, set the `stem` attribute in the document's header to `latexmath`. Equations and formulas are rendered using [KaTeX](https://katex.org/): ```plaintext diff --git a/doc/user/clusters/agent/ci_cd_workflow.md b/doc/user/clusters/agent/ci_cd_workflow.md index 644a753e282..c04c5a1f7ec 100644 --- a/doc/user/clusters/agent/ci_cd_workflow.md +++ b/doc/user/clusters/agent/ci_cd_workflow.md @@ -235,6 +235,10 @@ The identity can be specified with the following keys: See the [official Kubernetes documentation for details](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation). +## Related topics + +- [Self-paced classroom workshop](https://gitlab-for-eks.awsworkshop.io) (Uses AWS EKS, but you can use for other Kubernetes clusters) + ## Troubleshooting ### `kubectl` commands not supported diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index 6ca9d855b44..64eae308bec 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -65,7 +65,7 @@ gitops: - id: gitlab-org/cluster-integration/gitlab-agent default_namespace: my-ns paths: - # Read all YAML files from this directory. + # Read all YAML files from this directory. - glob: '/team1/app1/*.yaml' # Read all .yaml files from team2/apps and all subdirectories. - glob: '/team2/apps/**/*.yaml' @@ -124,10 +124,10 @@ As a result, every field in a resource can have different managers. Only fields are checked for drift. This facilitates the use of in-cluster controllers to modify resources like [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). -## Additional resources - -The following documentation and examples can help you get started with a GitOps workflow. +## Related topics +- [GitOps working examples for training and demos](https://gitlab.com/groups/guided-explorations/gl-k8s-agent/gitops/-/wikis/home) +- [Self-paced classroom workshop](https://gitlab-for-eks.awsworkshop.io) (Uses AWS EKS, but you can use for other Kubernetes clusters) - [Managing Kubernetes secrets in a GitOps workflow](gitops/secrets_management.md) - [Application and manifest repository example](https://gitlab.com/gitlab-examples/ops/gitops-demo/hello-world-service-gitops) diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index d54d432f0f5..5a69da28632 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -66,8 +66,11 @@ Read about how to [migrate to the agent for Kubernetes](../../infrastructure/clu ## Related topics - [GitOps workflow](gitops.md) +- [GitOps examples and learning materials](gitops.md#related-topics) - [GitLab CI/CD workflow](ci_cd_workflow.md) - [Install the agent](install/index.md) - [Work with the agent](repository.md) - [Troubleshooting](troubleshooting.md) +- [Guided explorations for a production ready GitOps setup](https://gitlab.com/groups/guided-explorations/gl-k8s-agent/gitops/-/wikis/home#gitlab-agent-for-kubernetes-gitops-working-examples) +- [CI/CD for Kubernetes examples and learning materials](ci_cd_workflow.md#related-topics) - [Contribute to the agent's development](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/doc) diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index f747c6c0e25..6c839f5ffc6 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -20,29 +20,51 @@ Before you can install the agent in your cluster, you need: - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/quickstart) - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) -- On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). Then it will be available by default at `wss://gitlab.example.com/-/kubernetes-agent/`. +- On self-managed GitLab instances, a GitLab administrator must set up the + [agent server](../../../../administration/clusters/kas.md). + Then it will be available by default at `wss://gitlab.example.com/-/kubernetes-agent/`. On GitLab.com, the agent server is available at `wss://kas.gitlab.com`. ## Installation steps To install the agent in your cluster: -1. [Choose a name for the agent](#agent-naming-convention). +1. Optional. [Create an agent configuration file](#create-an-agent-configuration-file). 1. [Register the agent with GitLab](#register-the-agent-with-gitlab). 1. [Install the agent in your cluster](#install-the-agent-in-the-cluster). <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> Watch a GitLab 14.2 [walk-through of this process](https://www.youtube.com/watch?v=XuBpKtsgGkE). -### Agent naming convention +### Create an agent configuration file + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. +> - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. + +The agent uses a YAML file for configuration settings. You must create this file if: + +- You use [a GitOps workflow](../gitops.md#gitops-workflow-steps). +- You use [a GitLab CI/CD workflow](../ci_cd_workflow.md#gitlab-cicd-workflow-steps) and want to authorize a different project to use the agent. + +To create an agent configuration file: + +1. Choose a name for your agent. The agent name follows the + [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). The name must: + + - Be unique in the project. + - Contain at most 63 characters. + - Contain only lowercase alphanumeric characters or `-`. + - Start with an alphanumeric character. + - End with an alphanumeric character. + +1. In the repository, create a directory in this location: -The agent name must follow the [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). -The name must: + ```plaintext + .gitlab/agents/<agent-name> + ``` -- Be unique in the project. -- Contain at most 63 characters. -- Contain only lowercase alphanumeric characters or `-`. -- Start with an alphanumeric character. -- End with an alphanumeric character. +1. In the directory, create a `config.yaml` file. Ensure the filename ends in `.yaml`, not `.yml`. + +You can leave the file blank for now, and [configure it](#configure-your-agent) later. ### Register the agent with GitLab @@ -64,34 +86,13 @@ You must register an agent before you can install the agent in your cluster. To it must be in this project. Your cluster manifest files should also be in this project. 1. From the left sidebar, select **Infrastructure > Kubernetes clusters**. 1. Select **Connect a cluster (agent)**. - - If you want to create a configuration with CI/CD defaults, type a name that meets [the naming convention](#agent-naming-convention). + - If you want to create a configuration with CI/CD defaults, type a name. - If you already have an [agent configuration file](#create-an-agent-configuration-file), select it from the list. 1. Select **Register an agent**. -1. GitLab generates an access token for the agent. Securely store this token. You need it to install the agent in your cluster and to [update the agent](#update-the-agent-version) to another version. -1. Copy the command under **Recommended installation method**. You need it when you use the one-liner installation method to install the agent in your cluster. - -### Create an agent configuration file - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7, the agent configuration file can be added to multiple directories (or subdirectories) of the repository. -> - Group authorization was [introduced](https://gitlab.com/groups/gitlab-org/-/epics/5784) in GitLab 14.3. - -The agent uses a YAML file for configuration settings. You need a configuration file if: - -- You want to use [a GitOps workflow](../gitops.md#gitops-configuration-reference). -- You want to authorize a different project to use the agent for a [GitLab CI/CD workflow](../ci_cd_workflow.md#authorize-the-agent). - -To create an agent configuration file: - -1. In the repository, create a directory in this location. The `<agent-name>` must meet [the naming convention](#agent-naming-convention). - - ```plaintext - .gitlab/agents/<agent-name> - ``` - -1. In the directory, create a `config.yaml` file. Ensure the filename ends in `.yaml`, not `.yml`. -1. Add content to the `config.yaml` file: - - For a GitOps workflow, view [the configuration reference](../gitops.md#gitops-configuration-reference) for details. - - For a GitLab CI/CD workflow, view [the configuration reference](../ci_cd_workflow.md) for details. +1. GitLab generates an access token for the agent. Securely store this token. You need it to install the agent + in your cluster and to [update the agent](#update-the-agent-version) to another version. +1. Copy the command under **Recommended installation method**. You need it when you use + the one-liner installation method to install the agent in your cluster. ### Install the agent in the cluster @@ -128,21 +129,45 @@ By default, the Helm installation command generated by GitLab: To see the full list of customizations available, see the Helm chart's [default values file](https://gitlab.com/gitlab-org/charts/gitlab-agent/-/blob/main/values.yaml). +##### Use the agent behind an HTTP proxy + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351867) in GitLab 15.0, the GitLab agent Helm chart supports setting environment variables. + +To configure an HTTP proxy when using the Helm chart, you can use the environment variables `HTTP_PROXY`, `HTTPS_PROXY`, +and `NO_PROXY`. Upper and lowercase are both acceptable. + +You can set these variables by using the `extraEnv` value, as a list of objects with keys `name` and `value`. +For example, to set only the environment variable `HTTPS_PROXY` to the value `https://example.com/proxy`, you can run: + +```shell +helm upgrade --install gitlab-agent gitlab/gitlab-agent \ + --set extraEnv[0].name=HTTPS_PROXY \ + --set extraEnv[0].value=https://example.com/proxy \ + ... +``` + #### Advanced installation method GitLab also provides a [KPT package for the agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/build/deployment/gitlab-agent). This method provides greater flexibility, but is only recommended for advanced users. +### Configure your agent + +To configure your agent, add content to the `config.yaml` file: + +- [View the configuration reference](../gitops.md#gitops-configuration-reference) for a GitOps workflow. +- [View the configuration reference](../ci_cd_workflow.md) for a GitLab CI/CD workflow. + ## Install multiple agents in your cluster To install a second agent in your cluster, you can follow the [previous steps](#register-the-agent-with-gitlab) a second time. To avoid resource name collisions within the cluster, you must either: -- Use a different release name for the agent, e.g. `second-gitlab-agent`: +- Use a different release name for the agent, for example, `second-gitlab-agent`: ```shell helm upgrade --install second-gitlab-agent gitlab/gitlab-agent ... ``` -- Or, install the agent in a different namespace, e.g. `different-namespace`: +- Or, install the agent in a different namespace, for example, `different-namespace`: ```shell helm upgrade --install gitlab-agent gitlab/gitlab-agent \ @@ -163,7 +188,7 @@ The following example projects can help you get started with the agent. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340882) in GitLab 14.8, GitLab warns you on the agent's list page to update the agent version installed on your cluster. -For the best experience, the version of the agent installed in your cluster should match the GitLab major and minor version. The previous minor version is also supported. For example, if your GitLab version is v14.9.4 (major version 14, minor version 9), then versions v14.9.0 and v14.9.1 of the agent are ideal, but any v14.8.x version of the agent is also supported. See [this page](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/releases) of releases of the GitLab agent. +For the best experience, the version of the agent installed in your cluster should match the GitLab major and minor version. The previous minor version is also supported. For example, if your GitLab version is v14.9.4 (major version 14, minor version 9), then versions v14.9.0 and v14.9.1 of the agent are ideal, but any v14.8.x version of the agent is also supported. See [the release page](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/releases) of the GitLab agent. ### Update the agent version diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md index 69f5b1d9063..706ed122f7b 100644 --- a/doc/user/clusters/agent/vulnerabilities.md +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Container vulnerability scanning **(ULTIMATE)** +# Operational Container Scanning **(ULTIMATE)** > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6346) in GitLab 14.8. @@ -16,8 +16,6 @@ You can also configure your agent so the vulnerabilities are displayed with othe Prerequisite: - You must have at least the Developer role. -- [Cluster image scanning](../../application_security/cluster_image_scanning/index.md) - must be part of your build process. To view vulnerability information in GitLab: @@ -28,6 +26,8 @@ To view vulnerability information in GitLab:  +This information can also be found under [operational vulnerabilities](../../../user/application_security/vulnerability_report/index.md#operational-vulnerabilities). + ## Enable cluster vulnerability scanning **(ULTIMATE)** You can use [cluster image scanning](../../application_security/cluster_image_scanning/index.md) @@ -39,8 +39,7 @@ containing a CRON expression for when the scans will be run. ```yaml starboard: - vulnerability_report: - cadence: '0 0 * * *' # Daily at 00:00 (Kubernetes cluster time) + cadence: '0 0 * * *' # Daily at 00:00 (Kubernetes cluster time) ``` The `cadence` field is required. GitLab supports the following types of CRON syntax for the cadence field: @@ -58,8 +57,8 @@ namespaces, you can use this configuration: ```yaml starboard: + cadence: '0 0 * * *' vulnerability_report: - cadence: '0 0 * * *' namespaces: - development - staging diff --git a/doc/user/clusters/create/index.md b/doc/user/clusters/create/index.md index bee622ac50a..b3d2b9f23fa 100644 --- a/doc/user/clusters/create/index.md +++ b/doc/user/clusters/create/index.md @@ -11,3 +11,4 @@ You connect the clusters to GitLab by using the agent for Kubernetes. - [Create a cluster on Google GKE](../../infrastructure/clusters/connect/new_gke_cluster.md) - [Create a cluster on Amazon EKS](../../infrastructure/clusters/connect/new_eks_cluster.md) +- [Create a cluster on Civo](../../infrastructure/clusters/connect/new_civo_cluster.md) diff --git a/doc/user/clusters/crossplane.md b/doc/user/clusters/crossplane.md index 3f38a473128..16615f88e25 100644 --- a/doc/user/clusters/crossplane.md +++ b/doc/user/clusters/crossplane.md @@ -10,4 +10,4 @@ redirect_to: '../../update/removals.md#managed-cluster-applicationsgitlab-ciyml' This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) -in GitLab 15.0. Use [crossplane](http://crossplane.io/) directly instead. +in GitLab 15.0. Use [crossplane](https://crossplane.io/) directly instead. diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index 4ba0de3bf55..b7732a7abf8 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -49,7 +49,7 @@ In order to: After you have successful deployments to your group-level or instance-level cluster: 1. Navigate to your group's **Kubernetes** page. -1. Click on the **Environments** tab. +1. Select the **Environments** tab. Only successful deployments to the cluster are included in this page. Non-cluster environments aren't included. diff --git a/doc/user/clusters/integrations.md b/doc/user/clusters/integrations.md index a6dbb5fe0d7..94fb443e0fb 100644 --- a/doc/user/clusters/integrations.md +++ b/doc/user/clusters/integrations.md @@ -98,7 +98,7 @@ To enable the Prometheus integration for your cluster: **Kubernetes** page. 1. Select the **Integrations** tab. 1. Check the **Enable Prometheus integration** checkbox. -1. Click **Save changes**. +1. Select **Save changes**. 1. Go to the **Health** tab to see your cluster's metrics. ## Elastic Stack cluster integration **(FREE SELF)** @@ -165,5 +165,5 @@ To enable the Elastic Stack integration for your cluster: **Kubernetes** page. 1. Select the **Integrations** tab. 1. Check the **Enable Elastic Stack integration** checkbox. -1. Click **Save changes**. +1. Select **Save changes**. 1. Go to the **Health** tab to see your cluster's metrics. diff --git a/doc/user/clusters/management_project_template.md b/doc/user/clusters/management_project_template.md index 8ca1bf5d57f..7ab77c67bcc 100644 --- a/doc/user/clusters/management_project_template.md +++ b/doc/user/clusters/management_project_template.md @@ -49,7 +49,7 @@ To create a project from the cluster management project template: 1. Select **Create project**. If you use self-managed GitLab, your instance might not include the latest version of the template. -In that case, select **Import project**, **Repo by URL** and for the **Git repository URL**, enter +In that case, select **Import project**, **Repository by URL** and for the **Git repository URL**, enter `https://gitlab.com/gitlab-org/project-templates/cluster-management.git`. ## Configure the project diff --git a/doc/user/compliance/compliance_report/index.md b/doc/user/compliance/compliance_report/index.md index 77dbefa0755..0006ae02752 100644 --- a/doc/user/compliance/compliance_report/index.md +++ b/doc/user/compliance/compliance_report/index.md @@ -113,7 +113,7 @@ You can generate a commit-specific Chain of Custody report for a given commit SH 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Compliance report**. -1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{angle-down}**). +1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{chevron-lg-down}**). 1. Enter the merge commit SHA, and then select **Export commit custody report**. SHA and then select **Export commit custody report**. diff --git a/doc/user/crm/index.md b/doc/user/crm/index.md index 7a400205e30..b5287816052 100644 --- a/doc/user/crm/index.md +++ b/doc/user/crm/index.md @@ -9,10 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2256) in GitLab 14.6 [with a flag](../../administration/feature_flags.md) named `customer_relations`. Disabled by default. > - In GitLab 14.8 and later, you can [create contacts and organizations only in root groups](https://gitlab.com/gitlab-org/gitlab/-/issues/350634). > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/346082) in GitLab 15.0. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `customer_relations`. -On GitLab.com, this feature is available. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/346082) in GitLab 15.1. With customer relations management (CRM) you can create a record of contacts (individuals) and organizations (companies) and relate them to issues. @@ -33,9 +30,13 @@ To read more about what is planned for the future, see [issue 2256](https://gitl ## Enable customer relations management (CRM) -To enable customer relations management in a group: +Customer relations management features must be enabled at the group level. If your +group also contains subgroups, and you want to use CRM features in the subgroup, +you must enable CRM features for the subgroup. -1. On the top bar, select **Menu > Groups** and find your group. +To enable customer relations management in a group or subgroup: + +1. On the top bar, select **Menu > Groups** and find your group or subgroup. 1. On the left sidebar, select **Settings > General**. 1. Expand the **Permissions and group features** section. 1. Select **Enable customer relations**. @@ -173,7 +174,6 @@ API. FLAG: On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `contacts_autocomplete`. On GitLab.com, this feature is available. -This feature is not ready for production use. When you use the `/add_contacts` or `/remove_contacts` quick actions, follow them with `[contact:` and an autocomplete list appears: diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 97c3d5f1058..a0649a61905 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -256,7 +256,7 @@ To create a thread by replying to a comment: The reply section is displayed. 1. Enter your reply. -1. Select **Comment** or **Add comment now** (depending on where in the UI you are replying). +1. Select **Reply** or **Add comment now** (depending on where in the UI you are replying). The top comment is converted to a thread. diff --git a/doc/user/free_user_limit.md b/doc/user/free_user_limit.md index 2340ef254f6..868e322cac9 100644 --- a/doc/user/free_user_limit.md +++ b/doc/user/free_user_limit.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Free user limit **(FREE SAAS)** -In GitLab 15.1 (June 22, 2022) and later, namespaces in GitLab.com on the Free tier +From September 15, 2022, namespaces in GitLab.com on the Free tier will be limited to five (5) members per [namespace](group/index.md#namespaces). This limit applies to top-level groups and personal namespaces. diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index e88777a7223..d515f9f4558 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -133,7 +133,7 @@ Below are the settings for [GitLab Pages](https://about.gitlab.com/stages-devops | IP address | `35.185.44.232` | - | | Custom domains support | **{check-circle}** Yes | **{dotted-circle}** No | | TLS certificates support | **{check-circle}** Yes | **{dotted-circle}** No | -| [Maximum size](../../administration/pages/index.md#set-global-maximum-pages-size-per-project) (compressed) | 1 GB | 100 MB | +| [Maximum size](../../administration/pages/index.md#set-global-maximum-size-of-each-gitlab-pages-site) (compressed) | 1 GB | 100 MB | The maximum size of your Pages site is also regulated by the artifacts maximum size, which is part of [GitLab CI/CD](#gitlab-cicd). @@ -156,10 +156,12 @@ the related documentation. | Maximum number of pipeline triggers in a project | `25000` for Free tier, Unlimited for all paid tiers | See [Limit the number of pipeline triggers](../../administration/instance_limits.md#limit-the-number-of-pipeline-triggers) | | Maximum pipeline schedules in projects | `10` for Free tier, `50` for all paid tiers | See [Number of pipeline schedules](../../administration/instance_limits.md#number-of-pipeline-schedules) | | Maximum pipelines per schedule | `24` for Free tier, `288` for all paid tiers | See [Limit the number of pipelines created by a pipeline schedule per day](../../administration/instance_limits.md#limit-the-number-of-pipelines-created-by-a-pipeline-schedule-per-day) | +| Maximum number of schedule rules defined for each security policy project | Unlimited for all paid tiers | See [Number of schedule rules defined for each security policy project](../../administration/instance_limits.md#limit-the-number-of-schedule-rules-defined-for-security-policy-project) | | Scheduled job archiving | 3 months (from June 22, 2020). Jobs created before that date were archived after September 22, 2020. | Never | -| Maximum test cases per [unit test report](../../ci/unit_test_reports.md) | `500000` | Unlimited | +| Maximum test cases per [unit test report](../../ci/testing/unit_test_reports.md) | `500000` | Unlimited | | Maximum registered runners | Free tier: `50` per-group / `50` per-project<br/>All paid tiers: `1000` per-group / `1000` per-project | See [Number of registered runners per scope](../../administration/instance_limits.md#number-of-registered-runners-per-scope) | | Limit of dotenv variables | Free tier: `50` / Premium tier: `100` / Ultimate tier: `150` | See [Limit dotenv variables](../../administration/instance_limits.md#limit-dotenv-variables) | +| Authorization token duration (minutes) | `15` | To set a custom value, in the Rails console, run `ApplicationSetting.last.update(container_registry_token_expire_delay: <integer>)`, where `<integer>` is the desired number of minutes. | ## Package registry limits @@ -186,7 +188,7 @@ the default value [is the same as for self-managed instances](../admin_area/sett | Setting | GitLab.com default | |-------------------------------|--------------------| | [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md#repository-size-limit) | 10 GB | -| Maximum import size | 5 GB | +| [Maximum import size](../project/settings/import_export.md#maximum-import-file-size) | 5 GB | | Maximum attachment size | 10 MB | If you are near or over the repository size limit, you can either @@ -232,7 +234,7 @@ The following limits apply for [webhooks](../project/integrations/webhooks.md): | Setting | Default for GitLab.com | |----------------------|-------------------------| -| Webhook rate limit | `120` calls per minute for GitLab Free, unlimited for GitLab Premium and GitLab Ultimate | +| Webhook rate limit | `500` calls per minute for GitLab Free, unlimited for GitLab Premium and GitLab Ultimate. Webhook rate limits are applied per top-level namespace. | | Number of webhooks | `100` per project, `50` per group | | Maximum payload size | 25 MB | @@ -328,19 +330,20 @@ limiting responses](#rate-limiting-responses). The following table describes the rate limits for GitLab.com, both before and after the limits change in January, 2021: -| Rate limit | From 2021-02-12 | From 2022-02-03 | -|:--------------------------------------------------------------------------|:------------------------------|:----------------------------------------| -| **Protected paths** (for a given **IP address**) | **10** requests per minute | **10** requests per minute | -| **Raw endpoint** traffic (for a given **project, commit, and file path**) | **300** requests per minute | **300** requests per minute | -| **Unauthenticated** traffic (from a given **IP address**) | **500** requests per minute | **500** requests per minute | -| **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | -| **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | -| **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | -| **Issue creation** | **300** requests per minute | **200** requests per minute | -| **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | -| **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | -| **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | -| **GitLab Pages** requests (for a given **GitLab Pages domain**) | | **5000** requests per **10 seconds** | +| Rate limit | From 2021-02-12 | From 2022-02-03 | +|:---------------------------------------------------------------------------|:------------------------------|:----------------------------------------| +| **Protected paths** (for a given **IP address**) | **10** requests per minute | **10** requests per minute | +| **Raw endpoint** traffic (for a given **project, commit, and file path**) | **300** requests per minute | **300** requests per minute | +| **Unauthenticated** traffic (from a given **IP address**) | **500** requests per minute | **500** requests per minute | +| **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | +| **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | +| **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | +| **Issue creation** | **300** requests per minute | **200** requests per minute | +| **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | +| **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | +| **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | +| **GitLab Pages** requests (for a given **GitLab Pages domain**) | | **5000** requests per **10 seconds** | +| **Pipeline creation** requests (for a given **project, user, and commit**) | | **25** requests per minute | More details are available on the rate limits for [protected paths](#protected-paths-throttle) and [raw @@ -424,7 +427,7 @@ setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/1 ### SSH maximum number of connections GitLab.com defines the maximum number of concurrent, unauthenticated SSH -connections by using the [MaxStartups setting](http://man.openbsd.org/sshd_config.5#MaxStartups). +connections by using the [MaxStartups setting](https://man.openbsd.org/sshd_config.5#MaxStartups). If more than the maximum number of allowed connections occur concurrently, they are dropped and users get [an `ssh_exchange_identification` error](../../topics/git/troubleshooting_git.md#ssh_exchange_identification-error). diff --git a/doc/user/group/contribution_analytics/index.md b/doc/user/group/contribution_analytics/index.md index cd2d5c190a1..ba805d6e1bb 100644 --- a/doc/user/group/contribution_analytics/index.md +++ b/doc/user/group/contribution_analytics/index.md @@ -49,7 +49,7 @@ Select the desired period from the calendar dropdown. ## Sorting by different factors -Contributions per group member are also presented in tabular format. Click a column header to sort the table by that column: +Contributions per group member are also presented in tabular format. Select a column header to sort the table by that column: - Member name - Number of pushed events diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index 00e6bc671c1..6755bf9ffb9 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -41,7 +41,7 @@ Projects in nested subgroups are not included in the template list. ## Which projects are available as templates - Public and internal projects can be selected by any signed-in user as a template for a new project, - if all [project features](../project/settings/index.md#sharing-and-permissions) + if all [project features](../project/settings/index.md#configure-project-visibility-features-and-permissions) except for **GitLab Pages** and **Security & Compliance** are set to **Everyone With Access**. - Private projects can be selected only by users who are members of the projects. diff --git a/doc/user/group/epics/img/epics_search_v14_7.png b/doc/user/group/epics/img/epics_filter_v14_7.png Binary files differindex baed532c736..baed532c736 100644 --- a/doc/user/group/epics/img/epics_search_v14_7.png +++ b/doc/user/group/epics/img/epics_filter_v14_7.png diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index e8f720238ed..e0334eda875 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -87,6 +87,26 @@ To edit an epic's start date, due date, or labels: 1. Next to each section in the right sidebar, select **Edit**. 1. Select the dates or labels for your epic. +### Reorder list items in the epic description + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.1. + +When you view an epic that has a list in the description, you can also reorder the list items. + +Prerequisites: + +- You must have at least the Reporter role for the project, be the author of the epic, or be + assigned to the epic. +- The epic's description must have an [ordered, unordered](../../markdown.md#lists), or + [task](../../markdown.md#task-lists) list. + +To reorder list items, when viewing an epic: + +1. Hover over the list item row to make the drag icon (**{drag-vertical}**) visible. +1. Select and hold the drag icon. +1. Drag the row to the new position in the list. +1. Release the drag icon. + ## Bulk edit epics > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7250) in GitLab 12.2. @@ -187,17 +207,17 @@ To view epics in a group: The total count of open epics displayed in the sidebar is cached if higher than 1000. The cached value is rounded to thousands or millions and updated every 24 hours. -## Search for an epic from epics list page +## Filter the list of epics -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) from GitLab Ultimate to GitLab Premium in 12.8. -> - Searching by the user's reaction emoji [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325630) in GitLab 13.11. +> - Filtering by epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195704) in GitLab 12.9. +> - Filtering by child epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9029) in GitLab 13.0. +> - Filtering by the user's reaction emoji [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325630) in GitLab 13.11. > - Sorting by epic titles [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.1. -> - Searching by milestone and confidentiality [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/268372) in GitLab 14.2 [with a flag](../../../administration/feature_flags.md) named `vue_epics_list`. Disabled by default. +> - Filtering by milestone and confidentiality [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/268372) in GitLab 14.2 [with a flag](../../../administration/feature_flags.md) named `vue_epics_list`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/276189) in GitLab 14.7. > - [Feature flag `vue_epics_list`](https://gitlab.com/gitlab-org/gitlab/-/issues/327320) removed in GitLab 14.8. -You can search for an epic from the list of epics using filtered search bar based on following -parameters: +You can filter the list of epics by: - Title or description - Author name / username @@ -206,9 +226,9 @@ parameters: - Confidentiality - Reaction emoji - + -To search: +To filter: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Epics**. @@ -216,7 +236,9 @@ To search: 1. From the dropdown menu, select the scope or enter plain text to search by epic title or description. 1. Press <kbd>Enter</kbd> on your keyboard. The list is filtered. -You can also sort epics list by: +## Sort the list of epics + +You can sort the epics list by: - Start date - Due date diff --git a/doc/user/group/img/restrict-by-email.gif b/doc/user/group/img/restrict-by-email.gif Binary files differdeleted file mode 100644 index d1ebeb07a0a..00000000000 --- a/doc/user/group/img/restrict-by-email.gif +++ /dev/null diff --git a/doc/user/group/img/restrict-by-ip.gif b/doc/user/group/img/restrict-by-ip.gif Binary files differdeleted file mode 100644 index 6292a58e748..00000000000 --- a/doc/user/group/img/restrict-by-ip.gif +++ /dev/null diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md index 6967815bf22..ae1465d0b1b 100644 --- a/doc/user/group/import/index.md +++ b/doc/user/group/import/index.md @@ -1,37 +1,42 @@ --- -type: reference, howto stage: Manage group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Migrate groups from another instance of GitLab **(FREE)** +# Migrating groups **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/249160) in GitLab 13.7 [with a flag](../../feature_flags.md) named `bulk_import`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338985) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/249160) in GitLab 13.7 for group resources [with a flag](../../feature_flags.md) named `bulk_import`. Disabled by default. +> - Group resources [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338985) in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 for project resources [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) named `bulk_import`. On GitLab.com, this feature is available. +On self-managed GitLab, by default [migrating group resources](#migrated-group-resources) is available. To hide the +feature, ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) named `bulk_import`. +On self-managed GitLab, by default [migrating project resources](#migrated-project-resources) is not available. To show +this feature, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named +`bulk_import_projects`. On GitLab.com, migrating group resources is available but migrating project resources is not +available. -You can migrate your existing top-level groups to any of the following: +Users with the Owner role on a top-level group can migrate it to: -- Another GitLab instance, including GitLab.com. - Another top-level group. - The subgroup of any existing top-level group. +- Another GitLab instance, including GitLab.com. -Migrating groups is not the same as [group import/export](../settings/import_export.md). - -- Group import/export requires you to export a group to a file and then import that file in - another GitLab instance. -- Group migration automates this process. +Migrating groups using the method documented here is not the same as [migrating groups using file exports](../settings/import_export.md). +Importing and exporting groups using file exports requires you to export a group to a file and then import that file in +another GitLab instance. Migrating groups using the method documented here automates this step. ## Import your groups into GitLab When you migrate a group, you connect to your GitLab instance and then choose -groups to import. Not all the data is migrated. View the -[Migrated resources](#migrated-resources) list for details. +groups to import. Not all the data is migrated. See: + +- [Migrated group resources](#migrated-group-resources). +- [Migrated project resources](#migrated-project-resources). -Leave feedback about group migration in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/284495). +Leave feedback about group migration in [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/284495). NOTE: You might need to reconfigure your firewall to prevent blocking the connection on the self-managed @@ -74,66 +79,57 @@ Migration importer page. The remote groups you have the Owner role for are liste For information on automating user, group, and project import API calls, see [Automate group and project import](../../project/import/index.md#automate-group-and-project-import). -## Migrated resources +## Migrated group resources Only the following resources are migrated to the target instance. Any other items are **not** migrated: -- Groups ([Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4374) in 13.7) - - description - - attributes - - subgroups - - avatar ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322904) in 14.0) -- Group labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292429) in 13.9) - - title - - description - - color - - created_at ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300007) in 13.10) - - updated_at ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300007) in 13.10) +- Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292431) in 13.11) +- Board Lists +- Boards +- Epics ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250281) in 13.7) +- Finisher +- Group Labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292429) in 13.9) +- Iterations ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292428) in 13.10) - Members ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299415) in 13.9) Group members are associated with the imported group if: - The user already exists in the target GitLab instance and - The user has a public email in the source GitLab instance that matches a confirmed email in the target GitLab instance -- Epics ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250281) in 13.7) - - title - - description - - state (open / closed) - - start date - - due date - - epic order on boards - - confidentiality - - labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/297460) in 13.9) - - author ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298745) in 13.9) - - parent epic ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/297459) in 13.9) - - emoji award ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/297466) in 13.9) - - events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/297465) in 13.10) - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292427) in 13.10) - - title - - description - - state (active / closed) - - start date - - due date - - created at - - updated at - - iid ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326157) in 13.11) -- Iterations ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292428) in 13.10) - - iid - - title - - description - - state (upcoming / started / closed) - - start date - - due date - - created at - - updated at -- Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292431) in 13.11) - - name - - link URL - - image URL -- Boards -- Board Lists +- Namespace Settings - Releases - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.0). +- Sub-Groups +- Uploads + +## Migrated project resources + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. + +FLAG: +On self-managed GitLab, migrating project resources are not available by default. To make them available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `bulk_import_projects`. On GitLab.com, migrating project resources are not available. + +- Projects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) + - Auto DevOps ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339410) in GitLab 14.6) + - Branches (including protected branches) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339414) in GitLab 14.7) + - CI Pipelines ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339407) in GitLab 14.6) + - Designs ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339421) in GitLab 15.1) + - Issues ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) in GitLab 14.4) + - Labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339419) in GitLab 14.4) + - LFS Objects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339405) in GitLab 14.8) + - Members ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341886) in GitLab 14.8) + - Merge Requests ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.5) + - Migrate Push Rules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.6) + - Pull Requests (including external pull requests) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339409) in GitLab 14.5) + - Pipeline History ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339412) in GitLab 14.6) + - Pipeline Schedules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339408) in GitLab 14.8) + - Releases ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.1) + - Release Evidences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360567) in GitLab 15.1) + - Repositories ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) + - Snippets ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343438) in GitLab 14.6) + - Uploads ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339401) in GitLab 14.5) + - Wikis ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345923) in GitLab 14.6) ## Troubleshooting Group Migration diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 87146329031..c0ae721e3b4 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -418,7 +418,7 @@ This action removes the group. It also adds a background job to delete all proje Specifically: -- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). - In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39504), if the user who sets up the deletion is removed from the group before the deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. @@ -599,7 +599,7 @@ You can export a list of members in a group or subgroup as a CSV. 1. Select **Export as CSV**. 1. After the CSV file has been generated, it is emailed as an attachment to the user that requested it. -## Restrict group access by IP address **(PREMIUM)** +## Group access restriction by IP address **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1985) in GitLab 12.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215410) from GitLab Ultimate to GitLab Premium in 13.1. @@ -611,24 +611,26 @@ applies to: - The GitLab UI, including subgroups, projects, and issues. - [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API. -You should consider these security implications before configuring IP address restrictions: - -- **SSH requests, including `git` operations will fail from all IP addresses**: While you can restrict HTTP traffic on GitLab.com with IP address restrictions, - they cause SSH requests, including Git operations over SSH, to fail. For more information, - read [issue 271673](https://gitlab.com/gitlab-org/gitlab/-/issues/271673). -- **Administrators and group owners can access group settings from any IP address**: Users with these permission levels can always - access the group settings, regardless of IP restriction, but they cannot access projects - belonging to the group when accessing from a disallowed IP address. -- **Some GitLab API endpoints will remain accessible from any IP**: Only the [group](../../api/groups.md) (including all - [group resources](../../api/api_resources.md#group-resources)) APIs and [project](../../api/api_resources.md#project-resources) - (including all [project resources](../../api/api_resources.md#project-resources)) APIs are protected by IP address restrictions. -- **Activities performed by GitLab Runners are not bound by IP restrictions**: - When you register a runner, it is not bound by the IP restrictions. When the runner - requests a new job or an update to a job's state, it is also not bound by - the IP restrictions. But when the running CI/CD job sends Git requests from a +### Security implications + +You should consider some security implications before configuring IP address restrictions. + +- Restricting HTTP traffic on GitLab.com with IP address restrictions causes SSH requests (including Git operations over + SSH) to fail. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/271673). +- Administrators and group owners can access group settings from any IP address, regardless of IP restriction. However: + - Groups owners cannot access projects belonging to the group when accessing from a disallowed IP address. + - Administrators can access projects belonging to the group when accessing from a disallowed IP address. + Access to projects includes cloning code from them. + - Users can still see group and project names and hierarchies. Only the following are restricted: + - [Groups](../../api/groups.md), including all [group resources](../../api/api_resources.md#group-resources). + - [Project](../../api/projects.md), including all [project resources](../../api/api_resources.md#project-resources). +- When you register a runner, it is not bound by the IP restrictions. When the runner requests a new job or an update to + a job's state, it is also not bound by the IP restrictions. But when the running CI/CD job sends Git requests from a restricted IP address, the IP restriction prevents code from being cloned. -- **User dashboard activity**: Users may still see some events from the IP restricted groups and projects - on their dashboard. Activity may include push, merge, issue, or comment events. +- Users may still see some events from the IP restricted groups and projects on their dashboard. Activity may include + push, merge, issue, or comment events. + +### Restrict group access by IP address To restrict group access by IP address: @@ -637,7 +639,9 @@ To restrict group access by IP address: 1. In the **Allow access to the following IP addresses** field, enter IPv4 or IPv6 address ranges in CIDR notation. 1. Select **Save changes**. -  +In self-managed installations of GitLab 15.1 and later, you can also configure +[globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges) +at the group level. ## Restrict group access by domain **(PREMIUM)** @@ -654,8 +658,6 @@ To restrict group access by domain: 1. In the **Restrict membership by email** field, enter the domain names. 1. Select **Save changes**. - - Any time you attempt to add a new user, the user's [primary email](../profile/index.md#change-your-primary-email) is compared against this list. Only users with a [primary email](../profile/index.md#change-your-primary-email) that matches any of the configured email domain restrictions can be added to the group. @@ -734,14 +736,17 @@ To disable group mentions: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. > - [Inheritance and enforcement added](https://gitlab.com/gitlab-org/gitlab/-/issues/321724) in GitLab 13.11. > - [Instance setting to enable by default added](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2. +> - [Instance setting is inherited and enforced when disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. +> - [User interface changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352961) in GitLab 15.1. -[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) can be enabled for groups. When enabled, projects in -the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. The default -period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#default-deletion-delay). +[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) is locked and disabled unless the instance-level settings for +[deletion protection](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) is enabled for either groups only or groups and projects. +When enabled on groups, projects in the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. +The default period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#retention-period). On self-managed GitLab, projects are deleted immediately by default. In GitLab 14.2 and later, an administrator can -[change the default setting](../admin_area/settings/visibility_and_access_controls.md#default-delayed-project-deletion) +[change the default setting](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) for projects in newly-created groups. On GitLab.com, see the [GitLab.com settings page](../gitlab_com/index.md#delayed-project-deletion) for @@ -751,8 +756,12 @@ To enable delayed deletion of projects in a group: 1. Go to the group's **Settings > General** page. 1. Expand the **Permissions and group features** section. -1. Check **Enable delayed project deletion**. -1. Optional. To prevent subgroups from changing this setting, select **Enforce for all subgroups**. +1. Scroll to: + - (GitLab 15.1 and later) **Deletion protection** and select **Keep deleted projects**. + - (GitLab 15.0 and earlier) **Enable delayed project deletion** and tick the checkbox. +1. Optional. To prevent subgroups from changing this setting, select: + - (GitLab 15.1 and later), **Enforce deletion protection for all subgroups** + - (GitLab 15.0 and earlier), **Enforce for all subgroups**. 1. Select **Save changes**. NOTE: @@ -766,8 +775,6 @@ By default, projects in a group can be forked. Optionally, on [GitLab Premium](https://about.gitlab.com/pricing/) or higher tiers, you can prevent the projects in a group from being forked outside of the current top-level group. -Previously, this setting was available only for groups enforcing a -[Group Managed Account](saml_sso/group_managed_accounts.md) in SAML. This setting will be removed from the SAML setting page, and migrated to the group settings page. In the interim period, both of these settings are taken into consideration. If even one is set to `true`, then the group does not allow outside forks. @@ -800,7 +807,7 @@ The group's new subgroups have push rules set for them based on either: - The closest parent group with push rules defined. - Push rules set at the instance level, if no parent groups have push rules defined. -## Group approval settings **(PREMIUM)** +## Group merge request approval settings **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285458) in GitLab 13.9. [Deployed behind the `group_merge_request_approval_settings_feature_flag` flag](../../administration/feature_flags.md), disabled by default. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) in GitLab 14.5. @@ -844,6 +851,7 @@ Support for group-level settings for merge request approval rules is tracked in - [Enforce two-factor authentication (2FA)](../../security/two_factor_authentication.md#enforce-2fa-for-all-users-in-a-group): Enforce 2FA for all group members. - Namespaces [API](../../api/namespaces.md) and [Rake tasks](../../raketasks/features.md). +- [Control access and visibility](../admin_area/settings/visibility_and_access_controls.md). ## Troubleshooting @@ -855,7 +863,7 @@ If a user sees a 404 when they would normally expect access, and the problem is - `json.allowed`: `false` In viewing the log entries, compare the `remote.ip` with the list of -[allowed IPs](#restrict-group-access-by-ip-address) for the group. +[allowed IPs](#group-access-restriction-by-ip-address) for the group. ### Validation errors on namespaces and groups diff --git a/doc/user/group/saml_sso/group_managed_accounts.md b/doc/user/group/saml_sso/group_managed_accounts.md index 6771ff8739a..0a00d0c1c1c 100644 --- a/doc/user/group/saml_sso/group_managed_accounts.md +++ b/doc/user/group/saml_sso/group_managed_accounts.md @@ -3,121 +3,12 @@ type: reference, howto stage: Manage group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-06-13' +redirect_to: 'index.md' --- # Group Managed Accounts **(PREMIUM)** -WARNING: -This [Closed Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#sts=Closed%20Beta) feature is being re-evaluated in favor of a different -[approach](https://gitlab.com/groups/gitlab-org/-/epics/4786) that aligns more closely with our [Subscription Agreement](https://about.gitlab.com/handbook/legal/subscription-agreement/). -We recommend that group owners who haven't yet implemented this feature wait for the new solution. - -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/709) in GitLab 12.1. -> - It's deployed behind a feature flag, disabled by default. - -When [SSO for Groups](index.md) is enforced, groups can enable an additional level of protection by enforcing the creation of dedicated user accounts to access the group. - -With group-managed accounts enabled, users are required to create a new, dedicated user linked to the group. -The notification email address associated with the user is locked to the email address received from the configured identity provider. -Without group-managed accounts, users can link their SAML identity with any existing user on the instance. - -When this option is enabled: - -- All users in the group are required to log in via the SSO URL associated with the group. -- After the group-managed account has been created, group activity requires the use of this user account. -- Users can't share a project in the group outside the top-level group (also applies to forked projects). - -Upon successful authentication, GitLab prompts the user with options, based on the email address received from the configured identity provider: - -- To create a unique account with the newly received email address. -- If the received email address matches one of the user's verified GitLab email addresses, the option to convert the existing account to a group-managed account. ([Introduced in GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/issues/13481).) - -Since use of the group-managed account requires the use of SSO, users of group-managed accounts lose access to these accounts when they are no longer able to authenticate with the connected identity provider. In the case of an offboarded employee who has been removed from your identity provider: - -- The user is unable to access the group (their credentials no longer work on the identity provider when prompted to use SSO). -- Contributions in the group (for example, issues and merge requests) remains intact. - -Please refer to our [SAML SSO for Groups page](../index.md) for information on how to configure SAML. - -## Feature flag **(PREMIUM SELF)** - -The group-managed accounts feature is behind these feature flags: `group_managed_accounts`, `sign_up_on_sso` and `convert_user_to_group_managed_accounts`. The flags are disabled by default. -To activate the feature, ask a GitLab administrator with Rails console access to run: - -```ruby -Feature.enable(:group_managed_accounts) -Feature.enable(:sign_up_on_sso) -Feature.enable(:convert_user_to_group_managed_accounts) -``` - -## Project restrictions for Group-managed accounts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12420) in GitLab 12.9. - -Projects within groups with enabled group-managed accounts are not to be shared with: - -- Groups outside of the parent group. -- Members who are not users managed by this group. - -This restriction also applies to projects forked from or to those groups. - -## Outer forks restriction for Group-managed accounts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34648) in GitLab 12.9. - -Groups with group-managed accounts can prevent forking of projects to destinations outside the group. -To do so, enable the "Prohibit outer forks" option in **Settings > SAML SSO**. -When enabled **at the parent group level**, projects within the group can be forked -only to other destinations within the group (including its subgroups). - -## Credentials inventory for Group-managed accounts **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38133) in GitLab 12.8. - -Owners who manage user accounts in a group can view the following details of personal access tokens and SSH keys: - -- Owners -- Scopes -- Usage patterns - -To access the Credentials inventory of a group, navigate to **{shield}** **Security & Compliance > Credentials** in your group's sidebar. - -This feature is similar to the [Credentials inventory for self-managed instances](../../admin_area/credentials_inventory.md). - -### Revoke a group-managed account's personal access token - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214811) in GitLab 13.5. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/267184) in GitLab 13.10. - -Group owners can revoke the personal access tokens of accounts in their group. To do so, select -the Personal Access Tokens tab, and select Revoke. - -When a personal access token is revoked, the group-managed account user is notified by email. - -## Limiting lifetime of personal access tokens of users in Group-managed accounts **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118893) in GitLab 12.10. - -Users in a group managed account can optionally specify an expiration date for -[personal access tokens](../../profile/personal_access_tokens.md). -This expiration date is not a requirement, and can be set to any arbitrary date. - -Since personal access tokens are the only token needed for programmatic access to GitLab, organizations with security requirements may want to enforce more protection to require regular rotation of these tokens. - -### Set a limit - -Only a GitLab administrator or an owner of a group-managed account can set a limit. When this field -is left empty, the [instance-level restriction](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) -on the lifetime of personal access tokens apply. - -To set a limit on how long personal access tokens are valid for users in a group managed account: - -1. Navigate to the **Settings > General** page in your group's sidebar. -1. Expand the **Permissions and group features** section. -1. Fill in the **Maximum allowable lifetime for access tokens (days)** field. -1. Click **Save changes**. - -Once a lifetime for personal access tokens is set: - -- GitLab applies the lifetime for new personal access tokens and requires users managed by the group to set an expiration date that's no later than the allowed lifetime. -- After three hours, revoke old tokens with no expiration date or with a lifetime longer than the allowed lifetime. Three hours is given to allow administrators/group owner to change the allowed lifetime, or remove it, before revocation takes place. +This [closed beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#sts=Closed%20Beta) feature was never enabled globally. See +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/296544) for progress on removing the feature. +Use [SAML SSO](index.md) instead. diff --git a/doc/user/group/saml_sso/group_sync.md b/doc/user/group/saml_sso/group_sync.md new file mode 100644 index 00000000000..2239562b831 --- /dev/null +++ b/doc/user/group/saml_sso/group_sync.md @@ -0,0 +1,161 @@ +--- +type: reference, howto +stage: Manage +group: Authentication and Authorization +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# SAML Group Sync **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363084) for self-managed instances in GitLab 15.1. + +WARNING: +Changing Group Sync configuration can remove users from the mapped GitLab group. +Removal happens if there is any mismatch between the group names and the list of `groups` in the SAML response. +If changes must be made, ensure either the SAML response includes the `groups` attribute +and the `AttributeValue` value matches the **SAML Group Name** in GitLab, +or that all groups are removed from GitLab to disable Group Sync. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For a demo of Group Sync using Azure, see [Demo: SAML Group Sync](https://youtu.be/Iqvo2tJfXjg). + +## Configure SAML Group Sync + +To configure SAML Group Sync: + +1. Configure SAML authentication: + - For GitLab self-managed, see [SAML OmniAuth Provider](../../../integration/saml.md). + - For GitLab.com, see [SAML SSO for GitLab.com groups](index.md). +1. Ensure your SAML identity provider sends an attribute statement named `Groups` or `groups`. + +NOTE: +The value for `Groups` or `groups` in the SAML response can be either the group name or the group ID. + +```xml +<saml:AttributeStatement> + <saml:Attribute Name="Groups"> + <saml:AttributeValue xsi:type="xs:string">Developers</saml:AttributeValue> + <saml:AttributeValue xsi:type="xs:string">Product Managers</saml:AttributeValue> + </saml:Attribute> +</saml:AttributeStatement> +``` + +Other attribute names such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/groups` +are not accepted as a source of groups. +See the [SAML troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md) +for examples on configuring the required attribute name in the SAML identity provider's settings. + +## Configure SAML Group Links + +When SAML is enabled, users with the Maintainer or Owner role +see a new menu item in group **Settings > SAML Group Links**. You can configure one or more **SAML Group Links** to map +a SAML identity provider group name to a GitLab role. This can be done for a top-level group or any subgroup. + +To link the SAML groups: + +1. In **SAML Group Name**, enter the value of the relevant `saml:AttributeValue`. +1. Choose the role in **Access Level**. +1. Select **Save**. +1. Repeat to add additional group links if required. + + + +If a user is a member of multiple SAML groups mapped to the same GitLab group, +the user gets the highest role from the groups. For example, if one group +is linked as Guest and another Maintainer, a user in both groups gets the Maintainer +role. + +Users granted: + +- A higher role with Group Sync are displayed as having + [direct membership](../../project/members/#display-direct-members) of the group. +- A lower or the same role with Group Sync are displayed as having + [inherited membership](../../project/members/#display-inherited-members) of the group. + +### Automatic member removal + +After a group sync, for GitLab subgroups, users who are not members of a mapped SAML +group are removed from the group. + +FLAG: +In [GitLab 15.1 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/364144), on GitLab.com, users in the top-level +group are assigned the [default membership role](index.md#role) rather than removed. This setting is enabled with the +`saml_group_sync_retain_default_membership` feature flag and can be configured by GitLab.com administrators only. + +For example, in the following diagram: + +- Alex Garcia signs into GitLab and is removed from GitLab Group C because they don't belong + to SAML Group C. +- Sidney Jones belongs to SAML Group C, but is not added to GitLab Group C because they have + not yet signed in. + +```mermaid +graph TB + subgraph SAML users + SAMLUserA[Sidney Jones] + SAMLUserB[Zhang Wei] + SAMLUserC[Alex Garcia] + SAMLUserD[Charlie Smith] + end + + subgraph SAML groups + SAMLGroupA["Group A"] --> SAMLGroupB["Group B"] + SAMLGroupA --> SAMLGroupC["Group C"] + SAMLGroupA --> SAMLGroupD["Group D"] + end + + SAMLGroupB --> |Member|SAMLUserA + SAMLGroupB --> |Member|SAMLUserB + + SAMLGroupC --> |Member|SAMLUserA + SAMLGroupC --> |Member|SAMLUserB + + SAMLGroupD --> |Member|SAMLUserD + SAMLGroupD --> |Member|SAMLUserC +``` + +```mermaid +graph TB + subgraph GitLab users + GitLabUserA[Sidney Jones] + GitLabUserB[Zhang Wei] + GitLabUserC[Alex Garcia] + GitLabUserD[Charlie Smith] + end + + subgraph GitLab groups + GitLabGroupA["Group A (SAML configured)"] --> GitLabGroupB["Group B (SAML Group Link not configured)"] + GitLabGroupA --> GitLabGroupC["Group C (SAML Group Link configured)"] + GitLabGroupA --> GitLabGroupD["Group D (SAML Group Link configured)"] + end + + GitLabGroupB --> |Member|GitLabUserA + + GitLabGroupC --> |Member|GitLabUserB + GitLabGroupC --> |Member|GitLabUserC + + GitLabGroupD --> |Member|GitLabUserC + GitLabGroupD --> |Member|GitLabUserD +``` + +```mermaid +graph TB + subgraph GitLab users + GitLabUserA[Sidney Jones] + GitLabUserB[Zhang Wei] + GitLabUserC[Alex Garcia] + GitLabUserD[Charlie Smith] + end + + subgraph GitLab groups after Alex Garcia signs in + GitLabGroupA[Group A] + GitLabGroupA["Group A (SAML configured)"] --> GitLabGroupB["Group B (SAML Group Link not configured)"] + GitLabGroupA --> GitLabGroupC["Group C (SAML Group Link configured)"] + GitLabGroupA --> GitLabGroupD["Group D (SAML Group Link configured)"] + end + + GitLabGroupB --> |Member|GitLabUserA + GitLabGroupC --> |Member|GitLabUserB + GitLabGroupD --> |Member|GitLabUserC + GitLabGroupD --> |Member|GitLabUserD +``` diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 1b480a52920..c05e847e2c9 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -176,6 +176,36 @@ If using [Group Sync](#group-sync), customize the name of the group claim to mat See the [troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory) for an example configuration. +### Google Workspace setup notes + +Follow the Google Workspace documentation on +[setting up SSO with Google as your identity provider](https://support.google.com/a/answer/6087519?hl=en) +with the notes below for consideration. + +| GitLab setting | Google Workspace field | +|:-------------------------------------|:-----------------------| +| Identifier | Entity ID | +| Assertion consumer service URL | ACS URL | +| GitLab single sign-on URL | Start URL | +| Identity provider single sign-on URL | SSO URL | + +You must download the certificate to get the SHA1 certificate fingerprint. + +The recommended attributes and claims settings are: + +- **Primary email** set to `email`. +- **First name** set to `first_name`. +- **Last name** set to `last_name`. + +For NameID, the following settings are recommended: + +- **Name ID format** is set to `EMAIL`. +- **NameID** set to `Basic Information > Primary email`. + +When selecting **Verify SAML Configuration** on the GitLab SAML SSO page, disregard the warning recommending setting the NameID format to "persistent". + +See the [troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md#google-workspace) for an example configuration. + ### Okta setup notes Please follow the Okta documentation on [setting up a SAML application in Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/main/) with the notes below for consideration. @@ -342,7 +372,7 @@ To rescind a user's access to the group when only SAML SSO is configured, either - Remove (in order) the user from: 1. The user data store on the identity provider or the list of users on the specific app. 1. The GitLab.com group. -- Use Group Sync at the top-level of your group to [automatically remove the user](#automatic-member-removal). +- Use Group Sync at the top-level of your group to [automatically remove the user](group_sync.md#automatic-member-removal). To rescind a user's access to the group when also using SCIM, refer to [Blocking access](scim_setup.md#blocking-access). @@ -372,149 +402,7 @@ For example, to unlink the `MyOrg` account: ## Group Sync -WARNING: -Changing Group Sync configuration can remove users from the relevant GitLab group. -Removal happens if there is any mismatch between the group names and the list of `groups` in the SAML response. -If changes must be made, ensure either the SAML response includes the `groups` attribute -and the `AttributeValue` value matches the **SAML Group Name** in GitLab, -or that all groups are removed from GitLab to disable Group Sync. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For a demo of Group Sync using Azure, see [Demo: SAML Group Sync](https://youtu.be/Iqvo2tJfXjg). - -When the SAML response includes a user and their group memberships from the SAML identity provider, -GitLab uses that information to automatically manage that user's GitLab group memberships. - -Ensure your SAML identity provider sends an attribute statement named `Groups` or `groups` like the following: - -```xml -<saml:AttributeStatement> - <saml:Attribute Name="Groups"> - <saml:AttributeValue xsi:type="xs:string">Developers</saml:AttributeValue> - <saml:AttributeValue xsi:type="xs:string">Product Managers</saml:AttributeValue> - </saml:Attribute> -</saml:AttributeStatement> -``` - -Other attribute names such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/groups` -are not accepted as a source of groups. -See the [SAML troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md) -for examples on configuring the required attribute name in the SAML identity provider's settings. - -NOTE: -The value for `Groups` or `groups` in the SAML response can be either the group name or the group ID. -To inspect the SAML response, you can use one of these [SAML debugging tools](#saml-debugging-tools). - -When SAML SSO is enabled for the top-level group, `Maintainer` and `Owner` level users -see a new menu item in group **Settings > SAML Group Links**. You can configure one or more **SAML Group Links** to map -a SAML identity provider group name to a GitLab Access Level. This can be done for the parent group or the subgroups. - -To link the SAML groups from the `saml:AttributeStatement` example above: - -1. In the **SAML Group Name** box, enter the value of `saml:AttributeValue`. -1. Choose the desired **Access Level**. -1. **Save** the group link. -1. Repeat to add additional group links if desired. - - - -If a user is a member of multiple SAML groups mapped to the same GitLab group, -the user gets the highest access level from the groups. For example, if one group -is linked as `Guest` and another `Maintainer`, a user in both groups gets `Maintainer` -access. - -Users granted: - -- A higher role with Group Sync are displayed as having - [direct membership](../../project/members/#display-direct-members) of the group. -- A lower or the same role with Group Sync are displayed as having - [inherited membership](../../project/members/#display-inherited-members) of the group. - -### Automatic member removal - -After a group sync, users who are not members of a mapped SAML group are removed from -the GitLab group. Even if SSO authentication is successful, if an existing user is not a member of any of the configured groups: - -- They get an "unauthorized" message if they try to view the group. -- All of their permissions to subgroups and projects are also removed. - -For example, in the following diagram: - -- Alex Garcia signs into GitLab and is removed from GitLab Group C because they don't belong - to SAML Group C. -- Sidney Jones belongs to SAML Group C, but is not added to GitLab Group C because they have - not yet signed in. - -```mermaid -graph TB - subgraph SAML users - SAMLUserA[Sidney Jones] - SAMLUserB[Zhang Wei] - SAMLUserC[Alex Garcia] - SAMLUserD[Charlie Smith] - end - - subgraph SAML groups - SAMLGroupA["Group A"] --> SAMLGroupB["Group B"] - SAMLGroupA --> SAMLGroupC["Group C"] - SAMLGroupA --> SAMLGroupD["Group D"] - end - - SAMLGroupB --> |Member|SAMLUserA - SAMLGroupB --> |Member|SAMLUserB - - SAMLGroupC --> |Member|SAMLUserA - SAMLGroupC --> |Member|SAMLUserB - - SAMLGroupD --> |Member|SAMLUserD - SAMLGroupD --> |Member|SAMLUserC -``` - -```mermaid -graph TB - subgraph GitLab users - GitLabUserA[Sidney Jones] - GitLabUserB[Zhang Wei] - GitLabUserC[Alex Garcia] - GitLabUserD[Charlie Smith] - end - - subgraph GitLab groups - GitLabGroupA["Group A (SAML configured)"] --> GitLabGroupB["Group B (SAML Group Link not configured)"] - GitLabGroupA --> GitLabGroupC["Group C (SAML Group Link configured)"] - GitLabGroupA --> GitLabGroupD["Group D (SAML Group Link configured)"] - end - - GitLabGroupB --> |Member|GitLabUserA - - GitLabGroupC --> |Member|GitLabUserB - GitLabGroupC --> |Member|GitLabUserC - - GitLabGroupD --> |Member|GitLabUserC - GitLabGroupD --> |Member|GitLabUserD -``` - -```mermaid -graph TB - subgraph GitLab users - GitLabUserA[Sidney Jones] - GitLabUserB[Zhang Wei] - GitLabUserC[Alex Garcia] - GitLabUserD[Charlie Smith] - end - - subgraph GitLab groups after Alex Garcia signs in - GitLabGroupA[Group A] - GitLabGroupA["Group A (SAML configured)"] --> GitLabGroupB["Group B (SAML Group Link not configured)"] - GitLabGroupA --> GitLabGroupC["Group C (SAML Group Link configured)"] - GitLabGroupA --> GitLabGroupD["Group D (SAML Group Link configured)"] - end - - GitLabGroupB --> |Member|GitLabUserA - GitLabGroupC --> |Member|GitLabUserB - GitLabGroupD --> |Member|GitLabUserC - GitLabGroupD --> |Member|GitLabUserD -``` +For information on automatically managing GitLab group membership, see [SAML Group Sync](group_sync.md). ## Passwords for users created via SAML SSO for Groups @@ -528,8 +416,8 @@ This section contains possible solutions for problems you might encounter. SAML responses are base64 encoded, so we recommend the following browser plugins to decode them on the fly: -- [SAML tracer for Firefox](https://addons.mozilla.org/en-US/firefox/addon/saml-tracer/) -- [Chrome SAML Panel](https://chrome.google.com/webstore/detail/saml-chrome-panel/paijfdbeoenhembfhkhllainmocckace?hl=en) +- [SAML-tracer](https://addons.mozilla.org/en-US/firefox/addon/saml-tracer/) for Firefox. +- [SAML Message Decoder](https://chrome.google.com/webstore/detail/saml-message-decoder/mpabchoaimgbdbbjjieoaeiibojelbhm?hl=en) for Chrome. Specific attention should be paid to: @@ -548,7 +436,7 @@ To generate a SAML Response: - [SAML-tracer](https://addons.mozilla.org/en-US/firefox/addon/saml-tracer/) for Firefox. 1. Open a new browser tab. 1. Open the SAML tracer console: - - Chrome: Right click on the page, select **Inspect**, then click on the SAML tab in the opened developer console. + - Chrome: Right-click on the page, select **Inspect**, then select the **SAML** tab in the opened developer console. - Firefox: Select the SAML-tracer icon located on the browser toolbar. 1. Go to the GitLab single sign-on URL for the group in the same browser tab with the SAML tracer open. 1. Select **Authorize** or attempt to log in. A SAML response is displayed in the tracer console that resembles this @@ -569,6 +457,10 @@ This can then be compared to the [NameID](#nameid) being sent by the identity pr ### Users receive a 404 +Because SAML SSO for groups is a paid feature, your subscription expiring can result in a `404` error when you're signing in using SAML SSO on GitLab.com. +If all users are receiving a `404` when attempting to log in using SAML, confirm +[there is an active subscription](../../../subscriptions/gitlab_com/index.md#view-your-gitlab-saas-subscription) being used in this SAML SSO namespace. + If you receive a `404` during setup when using "verify configuration", make sure you have used the correct [SHA-1 generated fingerprint](../../../integration/saml.md#notes-on-configuring-your-identity-provider). diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index bc1799c2e54..cc154b96ed0 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -82,7 +82,7 @@ table, use the Azure defaults. For a list of required attributes, refer to the [ For guidance, you can view [an example configuration in the troubleshooting reference](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory). -1. Below the mapping list click on **Show advanced options > Edit attribute list for AppName**. +1. Below the mapping list select **Show advanced options > Edit attribute list for AppName**. 1. Ensure the `id` is the primary and required field, and `externalId` is also required. NOTE: @@ -112,7 +112,7 @@ After the above steps are complete: 1. Sign in to Okta. 1. Ensure you are in the Admin Area by selecting the **Admin** button located in the top right. The button is not visible from the Admin Area. 1. In the **Application** tab, select **Browse App Catalog**. -1. Search for **GitLab**, find and select on the 'GitLab' application. +1. Search for **GitLab**, find and select the 'GitLab' application. 1. On the GitLab application overview page, select **Add**. 1. Under **Application Visibility** select both checkboxes. Currently the GitLab application does not support SAML authentication so the icon should not be shown to users. 1. Select **Done** to finish adding the application. diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index 4b791d5a221..649e7f2c264 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -36,7 +36,7 @@ You can use group access tokens: - Consider [disabling group access tokens](#enable-or-disable-group-access-token-creation) to lower potential abuse. -You cannot use group access tokens to create other access tokens. +You cannot use group access tokens to create other group, project, or personal access tokens. Group access tokens inherit the [default prefix setting](../../admin_area/settings/account_and_limit_settings.md#personal-access-token-prefix) configured for personal access tokens. @@ -147,9 +147,12 @@ Even when creation is disabled, you can still use and revoke existing group acce ## Bot users for groups -Each time you create a group access token, a bot user is created and added to the group. -These bot users are similar to [bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects), -except they are added to groups instead of projects. -These bot users do not count as licensed seats. +Each time you create a group access token, a bot user is created and added to the group. These bot users are similar to +[bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects), except they are added +to groups instead of projects. Bot users for groups: + +- Do not count as licensed seats. +- Can have a maximum role of Owner for a group. For more information, see + [Create a group access token](../../../api/group_access_tokens.md#create-a-group-access-token). For more information, see [Bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects). diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md index 23a638fb98c..eb94a181647 100644 --- a/doc/user/group/settings/import_export.md +++ b/doc/user/group/settings/import_export.md @@ -1,16 +1,24 @@ --- -type: reference stage: Manage group: Import info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Group import/export **(FREE)** +# Migrating groups using file exports (deprecated) **(FREE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. +> - [Deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6. -You can export groups, with all their related data, from one GitLab instance to another. -You can also [export projects](../../project/settings/import_export.md). +WARNING: +This feature was [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6 and replaced by +[a different migration method](../import/index.md). To follow progress on a solution for +[offline environments](../../application_security/offline_deployments/index.md), see +[the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/363406). + +You can export groups, with all their related data, from one GitLab instance to another. You can also: + +- [Migrate groups](../import/index.md) using the preferred method. +- [Migrate projects using file exports](../../project/settings/import_export.md). ## Enable export for a group @@ -63,10 +71,6 @@ For more details on the specific data persisted in a group export, see the ## Export a group -WARNING: -This feature will be [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) -in GitLab 14.6 and replaced by [GitLab Migration](../import/). - Prerequisites: - You must have the Owner role for the group. @@ -96,16 +100,11 @@ You can export groups from the [Community Edition to the Enterprise Edition](htt The Enterprise Edition retains some group data that isn't part of the Community Edition. If you're exporting a group from the Enterprise Edition to the Community Edition, you may lose this data. For more information, see [downgrading from EE to CE](../../../index.md). -## Importing the group - -WARNING: -This feature will be [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) -in GitLab 14.8 and replaced by [GitLab Migration](../import/). +## Import the group 1. Create a new group: - On the top bar, select **New** (**{plus}**) and then **New group**. - On an existing group's page, select the **New subgroup** button. - 1. Select **Import group**. 1. Enter your group name. 1. Accept or modify the associated group URL. diff --git a/doc/user/group/value_stream_analytics/index.md b/doc/user/group/value_stream_analytics/index.md index 4be97779603..72d42a8081f 100644 --- a/doc/user/group/value_stream_analytics/index.md +++ b/doc/user/group/value_stream_analytics/index.md @@ -50,9 +50,8 @@ To view value stream analytics for your group: - In the **To** field, select an end date. The charts and list show workflow items created during the date range. 1. Optional. Sort results by ascending or descending: - - To sort by most recent or oldest workflow item, select the **Merge requests** or **Issues** - header. The header name differs based on the stage you select. - - To sort by most or least amount of time spent in each stage, select the **Time** header. + - To sort by most recent or oldest workflow item, select the **Last event** header. + - To sort by most or least amount of time spent in each stage, select the **Duration** header. A badge next to the workflow items table header shows the number of workflow items that completed during the selected stage. @@ -366,7 +365,7 @@ The chart shows data for the last 500 workflow items. - In the **From** field, select a start date. - In the **To** field, select an end date. -## Type of work - Tasks by type chart +## Tasks by type chart > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32421) in GitLab 12.10. diff --git a/doc/user/infrastructure/clusters/connect/img/variables_civo.png b/doc/user/infrastructure/clusters/connect/img/variables_civo.png Binary files differnew file mode 100644 index 00000000000..5a20478b13c --- /dev/null +++ b/doc/user/infrastructure/clusters/connect/img/variables_civo.png diff --git a/doc/user/infrastructure/clusters/connect/index.md b/doc/user/infrastructure/clusters/connect/index.md index 004f4409919..37e1024a32c 100644 --- a/doc/user/infrastructure/clusters/connect/index.md +++ b/doc/user/infrastructure/clusters/connect/index.md @@ -10,11 +10,6 @@ The [certificate-based Kubernetes integration with GitLab](../index.md) was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. To connect your clusters, use the [GitLab agent](../../../clusters/agent/index.md). -<!-- TBA: (We need to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/343660 before adding this line) -If you don't have a cluster yet, create one and connect it to GitLab through the agent. -You can also create a new cluster from GitLab using [Infrastructure as Code](../../iac/index.md#create-a-new-cluster-through-iac). ---> - ## Cluster levels (DEPRECATED) > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/infrastructure/clusters/connect/new_civo_cluster.md b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md new file mode 100644 index 00000000000..d8401d5a286 --- /dev/null +++ b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md @@ -0,0 +1,137 @@ +--- +stage: Configure +group: Configure +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Create a Civo Kubernetes cluster + +Every new Civo account receives [$250 in credit](https://civo.com/signup) to get started with the GitLab integration with Civo Kubernetes. You can also use a marketplace app to install GitLab on your Civo Kubernetes cluster. + +Learn how to create a new cluster on Civo Kubernetes through +[Infrastructure as Code (IaC)](../../index.md). This process uses the Civo +and Kubernetes Terraform providers to create Civo Kubernetes clusters. You connect the clusters to GitLab +by using the GitLab agent for Kubernetes. + +**Prerequisites:** + +- A [Civo account](https://civo.com/signup). +- [A runner](https://docs.gitlab.com/runner/install/) you can use to run the GitLab CI/CD pipeline. + +**Steps:** + +1. [Import the example project](#import-the-example-project). +1. [Register the agent for Kubernetes](#register-the-agent). +1. [Configure your project](#configure-your-project). +1. [Provision your cluster](#provision-your-cluster). + +## Import the example project + +To create a cluster from GitLab using Infrastructure as Code, you must +create a project to manage the cluster from. In this tutorial, you start with +a sample project and modify it according to your needs. + +Start by [importing the example project by URL](../../../project/import/repo_by_url.md). + +To import the project: + +1. On the top bar, select **Menu > Create new project**. +1. Select **Import project**. +1. Select **Repository by URL**. +1. For the **Git repository URL**, enter `https://gitlab.com/civocloud/gitlab-terraform-civo.git`. +1. Complete the fields and select **Create project**. + +This project provides you with: + +- A [cluster on Civo](https://gitlab.com/civocloud/gitlab-terraform-civo/-/blob/master/civo.tf) with defaults for name, region, node count, and Kubernetes version. +- The [GitLab agent for Kubernetes](https://gitlab.com/civocloud/gitlab-terraform-civo/-/blob/master/agent.tf) installed in the cluster. + +## Register the agent + +To create a GitLab agent for Kubernetes: + +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. Select **Connect a cluster (agent)**. +1. From the **Select an agent** dropdown list, select `civo-agent` and select **Register an agent**. +1. GitLab generates a registration token for the agent. Securely store this secret token, as you will need it later. +1. GitLab provides an address for the agent server (KAS), which you will also need later. + +## Configure your project + +Use CI/CD environment variables to configure your project. + +**Required configuration:** + +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Variables**. +1. Set the variable `BASE64_CIVO_CREDENTIALS` to the [token](https://www.civo.com/account/security) from your Civo account. +1. Set the variable `TF_VAR_agent_token` to the agent token you received in the previous task. +1. Set the variable `TF_VAR_kas_address` to the agent server address in the previous task. + + + +**Optional configuration:** + +The file [`variables.tf`](https://gitlab.com/civocloud/gitlab-terraform-civo/-/blob/master/variables.tf) +contains other variables that you can override according to your needs: + +- `TF_VAR_civo_region`: Set your cluster's region. +- `TF_VAR_cluster_name`: Set your cluster's name. +- `TF_VAR_cluster_description`: Set a description for the cluster. To create a reference to your GitLab project on your Civo cluster detail page, set this value to `$CI_PROJECT_URL`. This value helps you determine which project was responsible for provisioning the cluster you see on the Civo dashboard. +- `TF_VAR_machine_type`: Set the machine type for the Kubernetes nodes. +- `TF_VAR_node_count`: Set the number of Kubernetes nodes. +- `TF_VAR_agent_version`: Set the version of the GitLab agent. +- `TF_VAR_agent_namespace`: Set the Kubernetes namespace for the GitLab agent. + +Refer to the [Civo Terraform provider](https://registry.terraform.io/providers/civo/civo/latest/docs/resources/kubernetes_cluster) and the [Kubernetes Terraform provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs) documentation for further resource options. + +## Provision your cluster + +After configuring your project, manually trigger the provisioning of your cluster. In GitLab: + +1. On the left sidebar, go to **CI/CD > Pipelines**. +1. Next to **Play** (**{play}**), select the dropdown icon (**{angle-down}**). +1. Select **Deploy** to manually trigger the deployment job. + +When the pipeline finishes successfully, you can see your new cluster: + +- In Civo dashboard: on your [Kubernetes tab](https://www.civo.com/account/kubernetes). +- In GitLab: from your project's sidebar, select **Infrastructure > Kubernetes clusters**. + +## Use your cluster + +After you provision the cluster, it is connected to GitLab and is ready for deployments. To check the connection: + +1. On the left sidebar, select **Infrastructure > Kubernetes clusters**. +1. In the list, view the **Connection status** column. + +For more information about the capabilities of the connection, see [the GitLab agent for Kubernetes documentation](../index.md). + +## Remove the cluster + +A cleanup job is not included in your pipeline by default. To remove all created resources, you +must modify your GitLab CI/CD template before running the cleanup job. + +To remove all resources: + +1. Add the following to your `.gitlab-ci.yml` file: + + ```yaml + stages: + - init + - validate + - build + - deploy + - cleanup + + destroy: + extends: .destroy + needs: [] + ``` + +1. On the left sidebar, select **CI/CD > Pipelines** and select the most recent pipeline. +1. For the `destroy` job, select **Play** (**{play}**). + +## Civo support + +This Civo integration is supported by Civo. Send your support requests to [Civo support](https://www.civo.com/contact). diff --git a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md index 50899053cad..798e02ab9b4 100644 --- a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md @@ -36,7 +36,7 @@ To import the project: 1. On the top bar, select **Menu > Create new project**. 1. Select **Import project**. -1. Select **Repo by URL**. +1. Select **Repository by URL**. 1. For the **Git repository URL**, enter `https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-eks.git`. 1. Complete the fields and select **Create project**. @@ -92,7 +92,7 @@ View the [AWS Terraform provider](https://registry.terraform.io/providers/hashic After configuring your project, manually trigger the provisioning of your cluster. In GitLab: 1. On the left sidebar, go to **CI/CD > Pipelines**. -1. Next to **Play** (**{play}**), select the dropdown icon (**{angle-down}**). +1. Next to **Play** (**{play}**), select the dropdown icon (**{chevron-lg-down}**). 1. Select **Deploy** to manually trigger the deployment job. When the pipeline finishes successfully, you can view the new cluster: diff --git a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md index be58b4565a1..07d0c722d8b 100644 --- a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md @@ -43,7 +43,7 @@ To import the project: 1. On the top bar, select **Menu > Create new project**. 1. Select **Import project**. -1. Select **Repo by URL**. +1. Select **Repository by URL**. 1. For the **Git repository URL**, enter `https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-gke.git`. 1. Complete the fields and select **Create project**. @@ -70,9 +70,6 @@ To create a GitLab agent for Kubernetes: To set up your project to communicate to GCP and the GitLab API: -1. Create a [GitLab personal access token](../../../profile/personal_access_tokens.md) with - `api` scope. The Terraform script uses it to connect the cluster to your GitLab group. Take note of the generated token. You will - need it when you [configure your project](#configure-your-project). 1. To authenticate GCP with GitLab, create a [GCP service account](https://cloud.google.com/docs/authentication/getting-started) with following roles: `Compute Network Viewer`, `Kubernetes Engine Admin`, `Service Account User`, and `Service Account Admin`. Both User and Admin service accounts are necessary. The User role impersonates the [default service account](https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) @@ -98,7 +95,7 @@ Use CI/CD environment variables to configure your project. 1. Set the variable `BASE64_GOOGLE_CREDENTIALS` to the `base64` encoded JSON file you just created. 1. Set the variable `TF_VAR_gcp_project` to your GCP's `project` name. 1. Set the variable `TF_VAR_agent_token` to the agent token displayed in the previous task. -1. Set the variable `TF_VAR_kas_address` to the agent server address displayed in the previous task. +1. Set the variable `TF_VAR_kas_address` to the agent server address displayed in the previous task. **Optional configuration:** @@ -120,7 +117,7 @@ Refer to the [Google Terraform provider](https://registry.terraform.io/providers After configuring your project, manually trigger the provisioning of your cluster. In GitLab: 1. On the left sidebar, go to **CI/CD > Pipelines**. -1. Next to **Play** (**{play}**), select the dropdown icon (**{angle-down}**). +1. Next to **Play** (**{play}**), select the dropdown icon (**{chevron-lg-down}**). 1. Select **Deploy** to manually trigger the deployment job. When the pipeline finishes successfully, you can see your new cluster: @@ -151,11 +148,12 @@ To remove all resources: - init - validate - build + - test - deploy - cleanup destroy: - extends: .destroy + extends: .terraform:destroy needs: [] ``` diff --git a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md index 7b2b5b4afd4..aa07a23db18 100644 --- a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md +++ b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md @@ -57,7 +57,7 @@ In your Auto DevOps project, you can use the GitLab agent to connect with your K - Add a key called `KUBE_INGRESS_BASE_DOMAIN` with the application deployment domain as the value. - Add a key called `KUBE_CONTEXT` with a value like `path/to/agent/project:agent-name`. Select the environment scope of your choice. - If you are not sure what your agent’s context is, edit your `.gitlab-ci.yml` file and add a job to see the available contexts: + If you are not sure what your agent's context is, edit your `.gitlab-ci.yml` file and add a job to see the available contexts: ```yaml deploy: diff --git a/doc/user/infrastructure/iac/index.md b/doc/user/infrastructure/iac/index.md index 0f7965a3527..422552c5b71 100644 --- a/doc/user/infrastructure/iac/index.md +++ b/doc/user/infrastructure/iac/index.md @@ -32,8 +32,7 @@ To get started, choose the template that best suits your needs: All templates: -- Use the [GitLab-managed Terraform state](#gitlab-managed-terraform-state) as - the Terraform state storage backend. +- Use the [GitLab-managed Terraform state](terraform_state.md) as the Terraform state storage backend. - Trigger four pipeline stages: `test`, `validate`, `build`, and `deploy`. - Run Terraform commands: `test`, `validate`, `plan`, and `plan-json`. It also runs the `apply` only on the default branch. - Run the [Terraform SAST scanner](../../application_security/iac_scanning/index.md#configure-iac-scanning-manually). @@ -58,6 +57,9 @@ to use one of these templates: - [The stable template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) with an skeleton that you can built on top of. - [The advanced template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml) to fully customize your setup. +NOTE: +In each GitLab major release (for example, 15.0), the latest templates replace the older ones. This process can introduce breaking changes. You can [use an older version of the template](troubleshooting.md#use-an-older-version-of-the-template) if you need to. + ### Use a Terraform template To use a Terraform template: @@ -86,37 +88,19 @@ To use a Terraform template: # TF_ROOT: terraform/production ``` -1. (Optional) Override in your `.gitlab-ci.yml` file the attributes present +1. Optional. Override in your `.gitlab-ci.yml` file the attributes present in the template you fetched to customize your configuration. -## GitLab-managed Terraform state - -Use the [GitLab-managed Terraform state](terraform_state.md) to store state -files in local storage or in a remote store of your choice. - -## Terraform module registry - -Use GitLab as a [Terraform module registry](../../packages/terraform_module_registry/index.md) -to create and publish Terraform modules to a private registry. - -## Terraform integration in merge requests - -Use the [Terraform integration in merge requests](mr_integration.md) -to collaborate on Terraform code changes and Infrastructure-as-Code -workflows. - -## The GitLab Terraform provider - -The [GitLab Terraform provider](https://github.com/gitlabhq/terraform-provider-gitlab) is a Terraform plugin to facilitate -managing of GitLab resources such as users, groups, and projects. It is released separately from GitLab -and its documentation is available on [Terraform](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs). - -## Create a new cluster through IaC - -- Learn how to [create a new cluster on Amazon Elastic Kubernetes Service (EKS)](../clusters/connect/new_eks_cluster.md). -- Learn how to [create a new cluster on Google Kubernetes Engine (GKE)](../clusters/connect/new_gke_cluster.md). - ## Related topics -- [Terraform images](https://gitlab.com/gitlab-org/terraform-images). -- [Troubleshooting](troubleshooting.md) issues with GitLab and Terraform. +- View [the images that contain the `gitlab-terraform` shell script](https://gitlab.com/gitlab-org/terraform-images). +- Use GitLab as a [Terraform module registry](../../packages/terraform_module_registry/index.md). +- To store state files in local storage or in a remote store, use the [GitLab-managed Terraform state](terraform_state.md). +- To collaborate on Terraform code changes and Infrastructure-as-Code workflows, use the + [Terraform integration in merge requests](mr_integration.md). +- To manage GitLab resources like users, groups, and projects, use the + [GitLab Terraform provider](https://github.com/gitlabhq/terraform-provider-gitlab). It is released separately from GitLab + and its documentation is available on [the Terraform docs site](https://registry.terraform.io/providers/gitlabhq/gitlab/latest/docs). +- [Create a new cluster on Amazon Elastic Kubernetes Service (EKS)](../clusters/connect/new_eks_cluster.md). +- [Create a new cluster on Google Kubernetes Engine (GKE)](../clusters/connect/new_gke_cluster.md). +- [Troubleshoot](troubleshooting.md) issues with GitLab and Terraform. diff --git a/doc/user/infrastructure/iac/terraform_state.md b/doc/user/infrastructure/iac/terraform_state.md index f56fe92ec01..e8637abce91 100644 --- a/doc/user/infrastructure/iac/terraform_state.md +++ b/doc/user/infrastructure/iac/terraform_state.md @@ -22,8 +22,13 @@ In GitLab, you can: - Lock and unlock states. - Remotely execute `terraform plan` and `terraform apply` commands. -For self-managed instances, before you can use GitLab for your Terraform state files, -an administrator must [set up Terraform state storage](../../../administration/terraform_state.md). +## Prerequisites + +For self-managed GitLab, before you can use GitLab for your Terraform state files: + +- An administrator must [set up Terraform state storage](../../../administration/terraform_state.md). +- You must enable the **Infrastructure** menu for your project. Go to **Settings > General**, + expand **Visibility, project features, permissions**, and under **Operations**, turn on the toggle. ## Initialize a Terraform state as a backend by using GitLab CI/CD @@ -56,7 +61,7 @@ To configure GitLab CI/CD as a backend: 1. In the root directory of your project repository, create a `.gitlab-ci.yml` file. Use [this file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) to populate it. - + 1. Push your project to GitLab. This action triggers a pipeline, which runs the `gitlab-terraform init`, `gitlab-terraform validate`, and `gitlab-terraform plan` commands. @@ -99,12 +104,31 @@ The following example demonstrates how to change the state name. The same workfl You can use a GitLab-managed Terraform state backend as a [Terraform data source](https://www.terraform.io/language/state/remote-state-data). -1. Create a file named `example.auto.tfvars` with the following contents: +1. In your `main.tf` or other relevant file, declare these variables. Leave the values empty. + + ```hcl + variable "example_remote_state_address" { + type = string + description = "Gitlab remote state file address" + } + + variable "example_username" { + type = string + description = "Gitlab username to query remote state" + } + + variable "example_access_token" { + type = string + description = "GitLab access token to query remote state" + } + ``` + +1. To override the values from the previous step, create a file named `example.auto.tfvars`. This file should **not** be versioned in your project repository. ```plaintext - example_remote_state_address=https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME> - example_username=<GitLab username> - example_access_token=<GitLab Personal Access Token> + example_remote_state_address = "https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>" + example_username = "<GitLab username>" + example_access_token = "<GitLab Personal Access Token>" ``` 1. In a `.tf` file, define the data source by using [Terraform input variables](https://www.terraform.io/language/values/variables): diff --git a/doc/user/infrastructure/iac/troubleshooting.md b/doc/user/infrastructure/iac/troubleshooting.md index 9dfe58396e2..881bcb32aed 100644 --- a/doc/user/infrastructure/iac/troubleshooting.md +++ b/doc/user/infrastructure/iac/troubleshooting.md @@ -60,7 +60,7 @@ my-terraform-job: extends: .apply ``` -There are two different causes for the error: +There are three different causes for the error: - In the case of `.init`, the error occurs because the init stage and jobs [were removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71188) from the templates, since they are no longer required. To resolve the syntax error, you can safely remove any jobs extending `.init`. - For all other jobs, the reason for the failure is that the base jobs have been [renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67719): A `.terraform:` prefix has been added to every job name. For example, `.apply` became `.terraform:apply`. To fix this error, you can update the base job names. For example: @@ -71,6 +71,21 @@ There are two different causes for the error: + extends: .terraform:apply ``` +- In GitLab 15.0, templates use [`rules`](../../../ci/yaml/index.md#rules) syntax + instead of [`only/except`](../../../ci/yaml/index.md#only--except). + Ensure the syntax in your `.gitlab-ci.yml` file does not include both. + +#### Use an older version of the template + +Breaking changes can occur during major releases. If you encounter a breaking change or want to use an older version of a template, you can update your `.gitlab-ci.yml` to refer to an older one. For example: + +```yaml +include: + remote: https://gitlab.com/gitlab-org/configure/template-archive/-/raw/main/14-10/Terraform.gitlab-ci.yml +``` + +View the [template-archive](https://gitlab.com/gitlab-org/configure/template-archive) to see which templates are available. + ## Troubleshooting Terraform state ### Unable to lock Terraform state files in CI jobs for `terraform apply` using a plan created in a previous job diff --git a/doc/user/markdown.md b/doc/user/markdown.md index bbcb0f62a5c..b1e6fcb2f4e 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -380,8 +380,8 @@ the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activati You can add task lists anywhere Markdown is supported. -- In issues, merge requests, and comments, you can click to select the boxes. -- In all other places, you cannot click to select the boxes. You must edit the Markdown manually +- In issues, merge requests, and comments, you can select the boxes. +- In all other places, you cannot select the boxes. You must edit the Markdown manually by adding or removing an `x` in the brackets. To create a task list, follow the format of an ordered or unordered list: @@ -790,12 +790,16 @@ do_this_and_do_that_and_another_thing but_emphasis is_desired _here_ ``` +<!-- vale gitlab.Spelling = NO --> + perform_complicated_task do_this_and_do_that_and_another_thing but_emphasis is_desired _here_ +<!-- vale gitlab.Spelling = YES --> + --- If you wish to emphasize only a part of a word, it can still be done with asterisks: diff --git a/doc/user/operations_dashboard/index.md b/doc/user/operations_dashboard/index.md index 20284328918..1b14582a3f0 100644 --- a/doc/user/operations_dashboard/index.md +++ b/doc/user/operations_dashboard/index.md @@ -20,12 +20,12 @@ have a [GitLab Premium](https://about.gitlab.com/pricing/) plan. To add a project to the dashboard: -1. Ensure your alerts - [populate the `gitlab_environment_name` field](../../operations/metrics/alerts.md#external-prometheus-instances). +1. Ensure your alerts populate the `gitlab_environment_name` label on the alerts you set up in Prometheus. + The value of this should match the name of your environment in GitLab. In GitLab 13.9, you can display alerts for the `production` environment only. -1. Click the **Add projects** button in the home screen of the dashboard. +1. Select **Add projects** in the home screen of the dashboard. 1. Search and add one or more projects using the **Search your projects** field. -1. Click the **Add projects** button. +1. Select **Add projects**. Once added, the dashboard displays the project's number of active alerts, last commit, pipeline status, and when it was last deployed. diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index fe7a41aa542..ae64c419632 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -57,7 +57,7 @@ To download and run a container image hosted in the GitLab Container Registry: 1. Copy the link to your container image: - Go to your project or group's **Packages & Registries > Container Registry** and find the image you want. - - Next to the image name, click the **Copy** button. + - Next to the image name, select **Copy**.  @@ -113,7 +113,7 @@ To authenticate, you can use: Both of these require the minimum scope to be: - For read (pull) access, `read_registry`. -- For write (push) access, `write_registry`. +- For write (push) access, `write_registry` & `read_registry`. To authenticate, run the `docker` command. For example: @@ -404,7 +404,7 @@ To delete images from within GitLab: by clicking the red **{remove}** **Trash** icon next to the tag you want to delete. -1. In the dialog box, click **Remove tag**. +1. In the dialog box, select **Remove tag**. ### Delete images using the API @@ -506,7 +506,7 @@ You can, however, remove the Container Registry for a project: 1. Go to your project's **Settings > General** page. 1. Expand the **Visibility, project features, permissions** section and disable **Container Registry**. -1. Click **Save changes**. +1. Select **Save changes**. The **Packages & Registries > Container Registry** entry is removed from the project's sidebar. @@ -707,3 +707,27 @@ GitLab is [migrating to the next generation of the Container Registry](https://g During the migration, you may encounter difficulty deleting tags. If you encounter an error, it's likely that your image repository is in the process of being migrated. Please wait a few minutes and try again. + +### `unauthorized: authentication required` when pushing large images + +When pushing large images, you might get an error like the following: + +```shell +docker push gitlab.example.com/myproject/docs:latest +The push refers to a repository [gitlab.example.com/myproject/docs] +630816f32edb: Preparing +530d5553aec8: Preparing +... +4b0bab9ff599: Waiting +d1c800db26c7: Waiting +42755cf4ee95: Waiting +unauthorized: authentication required +``` + +On self-managed GitLab instances, by default, tokens for the Container Registry expire every five minutes. +When pushing larger images, or images that take longer than five minutes to push, +you might encounter this error. On GitLab.com, the expiration time is 15 minutes. + +If you are using self-managed GitLab, you can ask an administrator to +[increase the token duration](../../../administration/packages/container_registry.md#increase-token-duration) +if necessary. diff --git a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md index 9dcb4d7127d..8f90f42ad08 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md +++ b/doc/user/packages/container_registry/reduce_container_registry_data_transfer.md @@ -114,8 +114,8 @@ up your builds and reduce the amount of data transferred. For more information, ## Check automation frequency -We often create automation scripts bundled into container images to perform regular tasks on specific intervals. -You can reduce the frequency of those intervals in cases where the automation is pulling container images from +We often create automation scripts bundled into container images to perform regular tasks on specific intervals. +You can reduce the frequency of those intervals in cases where the automation is pulling container images from the GitLab Registry to a service outside of GitLab.com. ## Move to GitLab Premium or Ultimate diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 7b52a6a8ce3..3e41a260854 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -109,12 +109,12 @@ To create a cleanup policy in the UI: | **Remove tags older than** | Remove only tags older than X days. | | **Remove tags matching** | The regex pattern that determines which tags to remove. This value cannot be blank. For all tags, use `.*`. See other [regex pattern examples](#regex-pattern-examples). | -1. Click **Save**. +1. Select **Save**. Depending on the interval you chose, the policy is scheduled to run. NOTE: -If you edit the policy and click **Save** again, the interval is reset. +If you edit the policy and select **Save** again, the interval is reset. ### Regex pattern examples diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md index 37e1f0c3eb1..22b792e443f 100644 --- a/doc/user/packages/generic_packages/index.md +++ b/doc/user/packages/generic_packages/index.md @@ -32,6 +32,11 @@ Prerequisites: If authenticating with a deploy token, it must be configured with the `write_package_registry` scope. If authenticating with a personal access token or project access token, it must be configured with the `api` scope. +- You must call this API endpoint serially when attempting to upload multiple files under the + same package name and version. Attempts to concurrently upload multiple files into + a new package name and version may face partial failures with + `HTTP 500: Internal Server Error` responses due to the requests racing to + create the package. ```plaintext PUT /projects/:id/packages/generic/:package_name/:package_version/:file_name?status=:status @@ -64,6 +69,14 @@ Example response without attribute `select`: } ``` +Example request with attribute `select = package_file`: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" \ + --upload-file path/to/file.txt \ + "https://gitlab.example.com/api/v4/projects/24/packages/generic/my_package/0.0.1/file.txt?select=package_file" +``` + Example response with attribute `select = package_file`: ```json diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index d2edfcb94c5..733fd383a04 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -108,7 +108,7 @@ and add the following text. Replace the variables in `< >` with your values. WARNING: If you use an environment variable called `NETRC`, Go will use its value -as a filename and ignore `~/.netrc`. If you intend to use `~/.netrc` in +as a filename and ignore `~/.netrc`. If you intend to use `~/.netrc` in the GitLab CI **do not use `NETRC` as an environment variable name**. ```plaintext diff --git a/doc/user/packages/helm_repository/index.md b/doc/user/packages/helm_repository/index.md index 73298afc9cd..88ea5afad3c 100644 --- a/doc/user/packages/helm_repository/index.md +++ b/doc/user/packages/helm_repository/index.md @@ -78,10 +78,10 @@ For example: ```yaml image: curlimages/curl:latest - + stages: - upload - + upload: stage: upload script: diff --git a/doc/user/packages/infrastructure_registry/index.md b/doc/user/packages/infrastructure_registry/index.md index 47f563fd7e7..551289a575a 100644 --- a/doc/user/packages/infrastructure_registry/index.md +++ b/doc/user/packages/infrastructure_registry/index.md @@ -15,19 +15,13 @@ projects. ## View packages -To view packages within your project or group: +To view packages within your project: -1. Go to the project or group. +1. Go to the project. 1. Go to **Packages & Registries > Infrastructure Registry**. You can search, sort, and filter packages on this page. -When you view packages in a group: - -- All packages published to the group and its projects are displayed. -- Only the projects you can access are displayed. -- If a project is private, or you are not a member of the project, it is not displayed. - For information on how to create and upload a package, view the GitLab documentation for your package type: @@ -68,7 +62,7 @@ To delete a package, you must have suitable [permissions](../../permissions.md). You can delete packages by using [the API](../../../api/packages.md#delete-a-project-package) or the UI. -To delete a package in the UI, from your group or project: +To delete a package in the UI, from your project: 1. Go to **Packages & Registries > Infrastructure Registry**. 1. Find the name of the package you want to delete. diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 1e3b5bdc323..eaa04404a1f 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -701,7 +701,7 @@ dependencies { For your project, go to **Packages & Registries > Package Registry**. -To remove a package, click the red trash icon or, from the package details, the **Delete** button. +To remove a package, select the red trash icon or, from the package details, the **Delete** button. ## Create Maven packages with GitLab CI/CD diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 1f37e5639c6..bdcbea68568 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -397,7 +397,7 @@ If multiple packages have the same name and version, when you install a package, - Replace `@foo` with your scope. - Replace `gitlab.example.com` with your domain name. - + For [project-level endpoints](#use-the-gitlab-endpoint-for-npm-packages) run: ```shell diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 37b6404d487..2e182d20811 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -171,7 +171,7 @@ To use the [project-level](#use-the-gitlab-endpoint-for-nuget-packages) NuGet en  -1. Click **Save**. +1. Select **Save**. The source is displayed in your list. @@ -200,7 +200,7 @@ To use the [group-level](#use-the-gitlab-endpoint-for-nuget-packages) NuGet endp  -1. Click **Save**. +1. Select **Save**. The source is displayed in your list. diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index 1f278bd1476..d68c7218ca5 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -65,7 +65,7 @@ For most package types, the following credential types are valid: [GitLab-#333444](https://gitlab.com/gitlab-org/gitlab/-/issues/333444), which prevents you from using a job token with internal projects. This bug only impacts self-managed GitLab instances. - + ## Use GitLab CI/CD to build packages You can use [GitLab CI/CD](../../../ci/index.md) to build packages. @@ -110,7 +110,7 @@ You can also remove the Package Registry for your project specifically: 1. In your project, go to **Settings > General**. 1. Expand the **Visibility, project features, permissions** section and disable the **Packages** feature. -1. Click **Save changes**. +1. Select **Save changes**. The **Packages & Registries > Package Registry** entry is removed from the sidebar. @@ -125,7 +125,7 @@ to **Only Project Members**, the Package Registry is then public. However, disab Registry disables all Package Registry operations. [GitLab-#329253](https://gitlab.com/gitlab-org/gitlab/-/issues/329253) -proposes adding the ability to control Package Registry visibility from the UI. +proposes adding the ability to control Package Registry visibility from the UI. | | | Anonymous<br/>(everyone on internet) | Guest | Reporter, Developer, Maintainer, Owner | | -------------------- | --------------------- | --------- | ----- | ------------------------------------------ | @@ -167,7 +167,7 @@ You can also use the [API](../../../api/packages.md) to administer the Package R ## Accepting contributions This table lists unsupported package manager formats that we are accepting contributions for. -Consider contributing to GitLab. This [development documentation](../../../development/packages.md) +Consider contributing to GitLab. This [development documentation](../../../development/packages/index.md) guides you through the process. <!-- vale gitlab.Spelling = NO --> diff --git a/doc/user/packages/package_registry/reduce_package_registry_storage.md b/doc/user/packages/package_registry/reduce_package_registry_storage.md index f8a1e63a228..ed4ef1665bc 100644 --- a/doc/user/packages/package_registry/reduce_package_registry_storage.md +++ b/doc/user/packages/package_registry/reduce_package_registry_storage.md @@ -31,7 +31,7 @@ To delete a package in the UI, from your group or project: 1. Go to **Packages & Registries > Package Registry**. 1. Find the name of the package you want to delete. -1. Click **Delete**. +1. Select **Delete**. The package is permanently deleted. diff --git a/doc/user/packages/terraform_module_registry/index.md b/doc/user/packages/terraform_module_registry/index.md index a107d06a63c..42c85ae9d41 100644 --- a/doc/user/packages/terraform_module_registry/index.md +++ b/doc/user/packages/terraform_module_registry/index.md @@ -22,11 +22,11 @@ To authenticate to the Terraform module registry, you need either: When you publish a Terraform Module, if it does not exist, it is created. -If a package with the same name and version already exists, it will not be created. It does not overwrite the existing package. - Prerequisites: -- You need to [authenticate with the API](../../../api/index.md#authentication). If authenticating with a deploy token, it must be configured with the `write_package_registry` scope. +- A package with the same name and version must not already exist. +- Your project and group names must not include a dot (`.`). For example, `source = "gitlab.example.com/my.group/project.name"`. +- You must [authenticate with the API](../../../api/index.md#authentication). If authenticating with a deploy token, it must be configured with the `write_package_registry` scope. ```plaintext PUT /projects/:id/packages/terraform/modules/:module-name/:module-system/:module-version/file @@ -35,8 +35,8 @@ PUT /projects/:id/packages/terraform/modules/:module-name/:module-system/:module | Attribute | Type | Required | Description | | -------------------| --------------- | ---------| -------------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../../../api/index.md#namespaced-path-encoding). | -| `module-name` | string | yes | The package name. It can contain only lowercase letters (`a-z`), uppercase letter (`A-Z`), numbers (`0-9`), or hyphens (`-`) and cannot exceed 64 characters. -| `module-system` | string | yes | The package system. It can contain only lowercase letters (`a-z`) and numbers (`0-9`), and cannot exceed 64 characters. More information can be found in the [Terraform Module Registry Protocol documentation](https://www.terraform.io/internals/module-registry-protocol). +| `module-name` | string | yes | The package name. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z), digits (0-9), and hyphens (`-`). +| `module-system` | string | yes | The package system. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z), digits (0-9), and hyphens (`-`). More information can be found in the [Terraform Module Registry Protocol documentation](https://www.terraform.io/internals/module-registry-protocol). | `module-version` | string | yes | The package version. It must be valid according to the [Semantic Versioning Specification](https://semver.org/). Provide the file content in the request body. @@ -111,17 +111,27 @@ To work with Terraform modules in [GitLab CI/CD](../../../ci/index.md), you can For example: ```yaml -image: curlimages/curl:latest - stages: - upload upload: stage: upload + image: curlimages/curl:latest + variables: + TERRAFORM_MODULE_DIR: ${CI_PROJECT_DIR} # The path to your Terraform module + TERRAFORM_MODULE_NAME: ${CI_PROJECT_NAME} # The name of your Terraform module + TERRAFORM_MODULE_SYSTEM: local # The system or provider your Terraform module targets (ex. local, aws, google) + TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version of your Terraform module to be published to your project's registry script: - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file path/to/file.tgz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/my-module/my-system/0.0.1/file"' + - tar -cvzf ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . + - 'curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/${TERRAFORM_MODULE_NAME}/${TERRAFORM_MODULE_SYSTEM}/${TERRAFORM_MODULE_VERSION}/file' + rules: + - if: $CI_COMMIT_TAG ``` +To trigger this upload job, add a Git tag to your commit. The `rules:if: $CI_COMMIT_TAG` defines this so that not every commit to your repo triggers the upload. +For other ways to control jobs in your CI/CD pipeline, refer to the [`.gitlab-ci.yml`](../../../ci/yaml/index.md) keyword reference. + ## Example projects For examples of the Terraform module registry, check the projects below: diff --git a/doc/user/permissions.md b/doc/user/permissions.md index d28cf75d11f..801c107e371 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -150,6 +150,7 @@ The following table lists project permissions available for each role: | [Projects](project/index.md):<br>View project [Audit Events](../administration/audit_events.md) | | | ✓ (*10*) | ✓ | ✓ | | [Projects](project/index.md):<br>Add [deploy keys](project/deploy_keys/index.md) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Add new [team members](project/members/index.md) | | | | ✓ | ✓ | +| [Projects](project/index.md):<br>Manage [team members](project/members/index.md) | | | | ✓ (*21*) | ✓ | | [Projects](project/index.md):<br>Change [project features visibility](public_access.md) level | | | | ✓ (*13*) | ✓ | | [Projects](project/index.md):<br>Configure [webhooks](project/integrations/webhooks.md) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Delete [wiki](project/wiki/index.md) pages | | | ✓ | ✓ | ✓ | @@ -157,7 +158,7 @@ The following table lists project permissions available for each role: | [Projects](project/index.md):<br>Edit project badges | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Edit project settings | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Export project | | | | ✓ | ✓ | -| [Projects](project/index.md):<br>Manage [project access tokens](project/settings/project_access_tokens.md) (*11*) | | | | ✓ | ✓ | +| [Projects](project/index.md):<br>Manage [project access tokens](project/settings/project_access_tokens.md) (*11*) | | | | ✓ (*21*) | ✓ | | [Projects](project/index.md):<br>Manage [Project Operations](../operations/index.md) | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Rename project | | | | ✓ | ✓ | | [Projects](project/index.md):<br>Share (invite) projects with groups | | | | ✓ (*7*) | ✓ (*7*) | @@ -212,7 +213,7 @@ The following table lists project permissions available for each role: public and internal projects (not on private projects). [External users](#external-users) must be given explicit access even if the project is internal. For GitLab.com, see the [GitLab.com visibility settings](gitlab_com/index.md#visibility-settings). -2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves. +2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to. 3. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [protected branches](project/protected_branches.md). 4. If the [branch is protected](project/protected_branches.md), this depends on the access Developers and Maintainers are given. 5. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see [repository information like commits and release evidence](project/releases/index.md#view-a-release-and-download-assets). @@ -232,11 +233,12 @@ The following table lists project permissions available for each role: 15. Guest users can only set metadata (for example, labels, assignees, or milestones) when creating an issue. They cannot change the metadata on existing issues. 16. In GitLab 14.5 or later, Guests are not allowed to [create incidents](../operations/incident_management/incidents.md#incident-creation). - A guest who created an incident when they had the Reporter role or who is assigned to the incident can modify the title, description and metrics. They can also close and reopen the incident. + In GitLab 15.1 and later, a Guest who created an issue that was promoted to an incident cannot edit, close, or reopen their incident. 17. In projects that accept contributions from external members, users can create, edit, and close their own merge requests. 18. Authors and assignees of issues can modify the title and description even if they don't have the Reporter role. 19. Authors and assignees can close and reopen issues even if they don't have the Reporter role. 20. The ability to view the Container Registry and pull images is controlled by the [Container Registry's visibility permissions](packages/container_registry/index.md#container-registry-visibility-permissions). +21. Maintainers cannot create, demote, or remove Owners, and they cannot promote users to the Owner role. They also cannot approve Owner role access requests. <!-- markdownlint-enable MD029 --> @@ -250,8 +252,8 @@ More details about the permissions for some project-level features follow. - [Public pipelines](../ci/pipelines/settings.md#change-which-users-can-view-your-pipelines): When set to public, gives access to certain CI/CD features to *Guest* project members. -- [Pipeline visibility](../ci/enable_or_disable_ci.md#enable-cicd-in-a-project): When set to **Everyone with Access**, - gives access to certain CI/CD "view" features to *non-project* members. +- [Pipeline visibility](../ci/pipelines/settings.md#change-pipeline-visibility-for-non-project-members-in-public-projects): + When set to **Everyone with Access**, gives access to certain CI/CD "view" features to *non-project* members. | Action | Non-member | Guest | Reporter | Developer | Maintainer | Owner | |---------------------------------------------------------------------------------------------------------------------------|------------|---------|----------|-----------|------------|-------| @@ -268,6 +270,7 @@ More details about the permissions for some project-level features follow. | Cancel and retry jobs | | | | ✓ | ✓ | ✓ | | Create new [environments](../ci/environments/index.md) | | | | ✓ | ✓ | ✓ | | Delete job logs or job artifacts | | | | ✓ (*4*) | ✓ | ✓ | +| Run CI/CD pipeline | | | | ✓ | ✓ | ✓ | | Run CI/CD pipeline for a protected branch | | | | ✓ (*5*) | ✓ (*5*) | ✓ | | Stop [environments](../ci/environments/index.md) | | | | ✓ | ✓ | ✓ | | View a job with [debug logging](../ci/variables/index.md#debug-logging) | | | | ✓ | ✓ | ✓ | @@ -355,7 +358,7 @@ Read through the documentation on [permissions for File Locking](project/file_lo ### Confidential Issues permissions [Confidential issues](project/issues/confidential_issues.md) can be accessed by users with reporter and higher permission levels, -as well as by guest users that create a confidential issue. To learn more, +as well as by guest users that create a confidential issue or are assigned to it. To learn more, read through the documentation on [permissions and access to confidential issues](project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues). ### Container Registry visibility permissions @@ -428,6 +431,7 @@ The following table lists group permissions available for each role: | View [Billing](../subscriptions/gitlab_com/index.md#view-your-gitlab-saas-subscription) | | | | | ✓ (4) | | View group [Usage Quotas](usage_quotas.md) page | | | | | ✓ (4) | | Manage group runners | | | | | ✓ | +| [Migrate groups](group/import/index.md) | | | | | ✓ | <!-- markdownlint-disable MD029 --> @@ -477,7 +481,7 @@ For example, if an external user is added as Guest, and your project is internal private, they do not have access to the code; you need to grant the external user access at the Reporter level or above if you want them to have access to the code. You should always take into account the -[project's visibility and permissions settings](project/settings/index.md#sharing-and-permissions) +[project's visibility and permissions settings](project/settings/index.md#configure-project-visibility-features-and-permissions) as well as the permission level of the user. NOTE: @@ -517,7 +521,7 @@ and the ignore case flag is set (`/regex pattern/i`). Here are some examples: - Use `\.internal@domain\.com$` to mark email addresses ending with `.internal@domain.com` as internal. - Use `^(?:(?!\.ext@domain\.com).)*$\r?` to mark users with email addresses - NOT including `.ext@domain.com` as internal. + not including `.ext@domain.com` as internal. WARNING: Be aware that this regex could lead to a diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index a820cf150e9..2dbeaae2267 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -22,7 +22,20 @@ If you set up a device, also set up a TOTP so you can still access your account ## Use personal access tokens with two-factor authentication When 2FA is enabled, you can't use your password to authenticate with Git over HTTPS or the [GitLab API](../../../api/index.md). -You must use a [personal access token](../personal_access_tokens.md) instead. +You can use a [personal access token](../personal_access_tokens.md) instead. + +## Git Credential Manager + +For Git over HTTPS, [Git Credential Manager](https://github.com/GitCredentialManager/git-credential-manager) (GCM) offers an alternative to personal access tokens. By default, GCM +authenticates using OAuth, opening GitLab in your web browser. The first time you authenticate, GitLab asks you to authorize the app. If you remain signed in to GitLab, subsequent +authentication requires no interaction. + +So you don't need to reauthenticate on every push, GCM supports caching as well as a variety of platform-specific credential stores that persist between sessions. This feature is useful whether you use personal access tokens or OAuth. + +GCM supports GitLab.com out the box. To use with self-managed GitLab, see [GitLab support](https://github.com/GitCredentialManager/git-credential-manager/blob/main/docs/gitlab.md) +documentation. + +Git Credential Manager is developed primarily by GitHub, Inc. It is an open-source project and is supported by the community. ## Enable two-factor authentication @@ -206,7 +219,7 @@ To set up 2FA with a U2F device: 1. Select **Account**. 1. Select **Enable Two-Factor Authentication**. 1. Connect your U2F device. -1. Select on **Set up New U2F Device**. +1. Select **Set up New U2F Device**. 1. A light begins blinking on your device. Activate it by pressing its button. A message displays indicating that your device was successfully set up. Select **Register U2F Device** to complete the @@ -322,9 +335,6 @@ To disable 2FA: This clears all your 2FA registrations, including mobile applications and U2F or WebAuthn devices. -Support Team support for disabling 2FA is limited, depending on your subscription level. For more information, see the -[Account Recovery](https://about.gitlab.com/support/#account-recovery-and-2fa-resets) section of our website. - ## Recovery options If you don't have access to your code generation device, you can recover access to your account: diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index d7eb5040416..07f21da3099 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -324,8 +324,12 @@ git config --global user.email <your email address> ## User activity -GitLab tracks user contribution activity. -You can follow or unfollow other users from their [user profiles](#access-your-user-profile). +GitLab tracks user contribution activity. You can follow or unfollow other users from either: + +- Their [user profiles](#access-your-user-profile). +- The small popover that appears when you hover over a user's name ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76050) + in GitLab 15.0). + To view a user's activity in a top-level Activity view: 1. From a user's profile, select **Follow**. @@ -395,7 +399,7 @@ You can add this URL to your feed reader. ### Reset the user activity feed token -Feed tokens are sensitive and can reveal information from confidential issues. +Feed tokens are sensitive and can reveal information from confidential issues. If you think your feed token has been exposed, you should reset it. To reset your feed token: @@ -404,7 +408,7 @@ To reset your feed token: 1. Select **Edit profile**. 1. On the left sidebar, select **Access Tokens**. 1. Scroll down. In the **Feed token** section, select the - **reset this token** link. + **reset this token** link. 1. On the confirmation box, select **OK**. A new token is generated. @@ -415,8 +419,13 @@ A new token is generated. When you sign in to the main GitLab application, a `_gitlab_session` cookie is set. When you close your browser, the cookie is cleared client-side -and it expires after "Application settings > Session duration (minutes)"/`session_expire_delay` -(defaults to `10080` minutes = 7 days) of no activity. +and it expires after a set duration. GitLab administrators can determine the duration: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Account and limit**. The set duration is in **Session duration (minutes)**. + +The default is `10080`, which equals 7 days. When you sign in to the main GitLab application, you can also check the **Remember me** option. This sets the `remember_user_token` diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 09e71ce9133..af84b746280 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -194,3 +194,8 @@ questions that you know someone might ask. Each scenario can be a third-level heading, e.g. `### Getting error message X`. If you have none to add when creating a doc, leave this section in place but commented out to help encourage others to add to it in the future. --> + +## Alternatives to personal access tokens + +For Git over HTTPS, an alternative to personal access tokens is [Git Credential Manager](account/two_factor_authentication.md#git-credential-manager), +which securely authenticates using OAuth. diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index c654eb2b0e8..e99a2dad980 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -91,7 +91,7 @@ Introduced in GitLab 13.6, the themes [Solarized](https://gitlab.com/gitlab-org/ ## Diff colors -A diff compares the old/removed content with the new/added content (e.g. when +A diff compares the old/removed content with the new/added content (for example, when [reviewing a merge request](../project/merge_requests/reviews/index.md#review-a-merge-request) or in a [Markdown inline diff](../markdown.md#inline-diff)). Typically, the colors red and green are used for removed and added lines in diffs. diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index 935bc01bae7..be73f2c9a01 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. WARNING: -This feature was deprecated in GitLab 14.5. Use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac) +This feature was deprecated in GitLab 14.5. Use [Infrastructure as Code](../../infrastructure/iac/index.md) to create new clusters. Through GitLab, you can create new clusters and add existing clusters hosted on Amazon Elastic @@ -23,7 +23,7 @@ use the [GitLab agent](../../clusters/agent/index.md). ## Create a new EKS cluster -To create a new cluster from GitLab, use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac). +To create a new cluster from GitLab, use [Infrastructure as Code](../../infrastructure/iac/index.md). ### How to create a new cluster on EKS through cluster certificates (DEPRECATED) @@ -58,7 +58,7 @@ cluster certificates: - Group's **Kubernetes** page, for a group-level cluster. - **Menu > Admin > Kubernetes**, for an instance-level cluster. 1. Select **Integrate with a cluster certificate**. -1. Under the **Create new cluster** tab, click **Amazon EKS** to display an +1. Under the **Create new cluster** tab, select **Amazon EKS** to display an `Account ID` and `External ID` needed for later steps. 1. In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create an IAM policy: 1. From the left panel, select **Policies**. @@ -116,8 +116,8 @@ cluster certificates: If you get an error during this process, GitLab does not roll back the changes. You must remove resources manually. You can do this by deleting the relevant [CloudFormation stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-delete-stack.html). - 1. Click **Review policy**. - 1. Enter a suitable name for this policy, and click **Create Policy**. You can now close this window. + 1. Select **Review policy**. + 1. Enter a suitable name for this policy, and select **Create Policy**. You can now close this window. ### Prepare the cluster in Amazon @@ -140,16 +140,16 @@ In the [IAM Management Console](https://console.aws.amazon.com/iam/home), create another IAM role (**role B**) for GitLab authentication with AWS: 1. On the AWS IAM console, select **Roles** from the left panel. -1. Click **Create role**. +1. Select **Create role**. 1. Under **Select type of trusted entity**, select **Another AWS account**. 1. Enter the Account ID from GitLab into the **Account ID** field. 1. Check **Require external ID**. 1. Enter the External ID from GitLab into the **External ID** field. -1. Click **Next: Permissions**, and select the policy you just created. -1. Click **Next: Tags**, and optionally enter any tags you wish to associate with this role. -1. Click **Next: Review**. +1. Select **Next: Permissions**, and select the policy you just created. +1. Select **Next: Tags**, and optionally enter any tags you wish to associate with this role. +1. Select **Next: Review**. 1. Enter a role name and optional description into the fields provided. -1. Click **Create role**. The new role name displays at the top. Click on its name and copy the +1. Select **Create role**. The new role name displays at the top. Select its name and copy the `Role ARN` from the newly created role. ### Configure your cluster's data in GitLab @@ -213,7 +213,7 @@ Otherwise, the deployed app isn't externally available outside of the cluster. GitLab creates a new pipeline, which begins to build, test, and deploy the app. After the pipeline has finished, your app runs in EKS, and is available -to users. Click on **CI/CD > Environments**. +to users. Select **CI/CD > Environments**.  @@ -252,7 +252,7 @@ IAM user in the Amazon AWS console, and follow these steps: 1. Check **Enable Amazon EKS integration**. 1. Enter your **Account ID**. 1. Enter your [access key and ID](#eks-access-key-and-id). -1. Click **Save changes**. +1. Select **Save changes**. #### EKS access key and ID diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index cb8d04e0e28..2e1c8766ae3 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -64,8 +64,8 @@ cluster certificates: cluster. - Group's **{cloud-gear}** **Kubernetes** page, for a group-level cluster. - **Menu > Admin > Kubernetes** page, for an instance-level cluster. -1. Click **Integrate with a cluster certificate**. -1. Under the **Create new cluster** tab, click **Google GKE**. +1. Select **Integrate with a cluster certificate**. +1. Under the **Create new cluster** tab, select **Google GKE**. 1. Connect your Google account if you haven't done already by clicking the **Sign in with Google** button. 1. Choose your cluster's settings: @@ -83,7 +83,7 @@ cluster certificates: See the [Cloud Run for Anthos section](#cloud-run-for-anthos) for more information. - **GitLab-managed cluster** - Leave this checked if you want GitLab to manage namespaces and service accounts for this cluster. See the [Managed clusters section](gitlab_managed_clusters.md) for more information. -1. Finally, click the **Create Kubernetes cluster** button. +1. Finally, select the **Create Kubernetes cluster** button. After a couple of minutes, your cluster is ready. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index 8cdd1792e7f..95d8064b380 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 14.0. -To create and manage a new cluster use [Infrastructure as Code](../../infrastructure/iac/index.md#create-a-new-cluster-through-iac). +To create and manage a new cluster use [Infrastructure as Code](../../infrastructure/iac/index.md). ## Disable a cluster @@ -22,7 +22,7 @@ When you successfully connect an existing cluster using cluster certificates, th - **Menu > Admin > Kubernetes** page, for an instance-level cluster. 1. Select the name of the cluster you want to disable. 1. Toggle **GitLab Integration** off (in gray). -1. Click **Save changes**. +1. Select **Save changes**. ## Remove a cluster diff --git a/doc/user/project/clusters/gitlab_managed_clusters.md b/doc/user/project/clusters/gitlab_managed_clusters.md index e295abf8d31..3b85d29fb4a 100644 --- a/doc/user/project/clusters/gitlab_managed_clusters.md +++ b/doc/user/project/clusters/gitlab_managed_clusters.md @@ -47,7 +47,7 @@ To clear the cache: 1. Navigate to your project's **Infrastructure > Kubernetes clusters** page, and select your cluster. 1. Expand the **Advanced settings** section. -1. Click **Clear cluster cache**. +1. Select **Clear cluster cache**. ## Base domain @@ -66,10 +66,10 @@ You can either: To determine the external Ingress IP address, or external Ingress hostname: - *If the cluster is on GKE*: - 1. Click the **Google Kubernetes Engine** link in the **Advanced settings**, + 1. Select the **Google Kubernetes Engine** link in the **Advanced settings**, or go directly to the [Google Kubernetes Engine dashboard](https://console.cloud.google.com/kubernetes/). 1. Select the proper project and cluster. - 1. Click **Connect** + 1. Select **Connect**. 1. Execute the `gcloud` command in a local terminal or using the **Cloud Shell**. - *If the cluster is not on GKE*: Follow the specific instructions for your diff --git a/doc/user/project/clusters/kubernetes_pod_logs.md b/doc/user/project/clusters/kubernetes_pod_logs.md index b1158be9fb6..58006c29057 100644 --- a/doc/user/project/clusters/kubernetes_pod_logs.md +++ b/doc/user/project/clusters/kubernetes_pod_logs.md @@ -52,7 +52,7 @@ is required to use Logs. ## Accessing the log explorer -To access the **Log explorer**, click the **More actions** **{ellipsis_v}** menu on +To access the **Log explorer**, select the **More actions** **{ellipsis_v}** menu on a [metrics dashboard](../../../operations/metrics/index.md) and select **View logs**, or: 1. Sign in as a user with the _View pod logs_ @@ -68,7 +68,7 @@ a [metrics dashboard](../../../operations/metrics/index.md) and select **View lo 1. When mousing over the list of pods, GitLab displays a tooltip with the exact pod name and status.  - 1. Click on the desired pod to display the **Log Explorer**. + 1. Select the desired pod to display the **Log Explorer**. ### Logs view @@ -97,7 +97,7 @@ Support for historical data is coming When you enable [Elastic Stack](../../clusters/integrations.md#elastic-stack-cluster-integration) on your cluster, you can filter logs displayed in the **Log Explorer** by date. -Click **Show last** in the **Log Explorer** to see the available options. +Select **Show last** in the **Log Explorer** to see the available options. ### Full text search diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index 086e1fccf6c..94b5f6f52b9 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -55,7 +55,7 @@ Nurtch is the company behind the [Rubix library](https://github.com/Nurtch/rubix Rubix is an open-source Python library that makes it easy to perform common DevOps tasks inside Jupyter Notebooks. Tasks such as plotting Cloudwatch metrics and rolling your ECS/Kubernetes app are simplified down to a couple of lines of -code. See the [Nurtch Documentation](http://docs.nurtch.com/en/latest/) for more +code. See the [Nurtch Documentation](https://docs.nurtch.com/en/latest/) for more information. ## Configure an executable runbook with GitLab @@ -153,15 +153,15 @@ the components outlined above and the pre-loaded demo runbook. ``` 1. After JupyterHub has been installed successfully, open the **Jupyter Hostname** - in your browser. Click the **Sign in with GitLab** button to log in to + in your browser. Select **Sign in with GitLab** button to log in to JupyterHub and start the server. Authentication is enabled for any user of the GitLab instance with OAuth2. This button redirects you to a page at GitLab requesting authorization for JupyterHub to use your GitLab account.  -1. Click **Authorize**, and GitLab redirects you to the JupyterHub application. -1. Click **Start My Server** to start the server in a few seconds. +1. Select **Authorize**, and GitLab redirects you to the JupyterHub application. +1. Select **Start My Server** to start the server in a few seconds. 1. To configure the runbook's access to your GitLab project, you must enter your [GitLab Access Token](../../../profile/personal_access_tokens.md) and your Project ID in the **Setup** section of the demo runbook: @@ -208,13 +208,13 @@ the components outlined above and the pre-loaded demo runbook.  - 1. Click **Save variables**. + 1. Select **Save variables**. - 1. In Jupyter, click the **Run SQL queries in Notebook** heading, and then click + 1. In Jupyter, select the **Run SQL queries in Notebook** heading, and then select **Run**. The results are displayed inline as follows:  You can try other operations, such as running shell scripts or interacting with a Kubernetes cluster. Visit the -[Nurtch Documentation](http://docs.nurtch.com/) for more information. +[Nurtch Documentation](https://docs.nurtch.com/) for more information. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index e37ff560080..197a995952a 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -255,7 +255,7 @@ README @group @group/with-nested/subgroup /docs/* @root-docs # Include `/**` to specify Code Owners for all subdirectories -# in a directory. This rule matches `docs/projects/index.md` or +# in a directory. This rule matches `docs/projects/index.md` or # `docs/development/index.md` /docs/**/*.md @root-docs diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index 42c1b8b0a62..a90ffbe5796 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -69,7 +69,7 @@ specific environment, there are a lot of use cases. To name a few: - You want to promote what's running in staging, to production. You go to the environments list, verify that what's running in staging is what you think is - running, then click on the [manual job](../../ci/jobs/job_control.md#create-a-job-that-must-be-run-manually) to deploy to production. + running, then select the [manual job](../../ci/jobs/job_control.md#create-a-job-that-must-be-run-manually) to deploy to production. - You trigger a deploy, and you have many containers to upgrade so you know this takes a while (you've also throttled your deploy to only take down X containers at a time). But you need to tell someone when it's deployed, so you @@ -80,7 +80,7 @@ specific environment, there are a lot of use cases. To name a few: stuck or failed. - You've got an MR that looks good, but you want to run it on staging because staging is set up in some way closer to production. You go to the environment - list, find the [Review App](../../ci/review_apps/index.md) you're interested in, and click the + list, find the [Review App](../../ci/review_apps/index.md) you're interested in, and select the manual action to deploy it to staging. ## Enabling deploy boards @@ -129,7 +129,7 @@ To display the deploy boards for a specific [environment](../../ci/environments/ Once all of the above are set up and the pipeline has run at least once, navigate to the environments page under **Deployments > Environments**. -Deploy boards are visible by default. You can explicitly click +Deploy boards are visible by default. You can explicitly select the triangle next to their respective environment name in order to hide them. ### Example manifest file diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index 64c18ab6f3b..595f5e541b7 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -190,6 +190,8 @@ To pull images from the Dependency Proxy, you must: ### GitLab deploy token +> Support for `gitlab-deploy-token` at the group level [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214014) in GitLab 15.1 [with a flag](../../../administration/feature_flags.md) named `ci_variable_for_group_gitlab_deploy_token`. Enabled by default. + There's a special case when it comes to deploy tokens. If a user creates one named `gitlab-deploy-token`, the username and token of the deploy token is automatically exposed to the CI/CD jobs as CI/CD variables: `CI_DEPLOY_USER` @@ -203,9 +205,10 @@ docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY ``` NOTE: -The special handling for the `gitlab-deploy-token` deploy token is not -implemented for group deploy tokens. To make the group-level deploy token available for -CI/CD jobs, the `CI_DEPLOY_USER` and `CI_DEPLOY_PASSWORD` variables should be set under **Settings** to the name and token of the group deploy token respectively. +In GitLab 15.0 and earlier, the special handling for the `gitlab-deploy-token` deploy token +does not work for group deploy tokens. To make the group-level deploy token available +for CI/CD jobs, the `CI_DEPLOY_USER` and `CI_DEPLOY_PASSWORD` CI/CD variables must be +set in **Settings > CI/CD > Variables** to the name and token of the group deploy token. ## Troubleshooting diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index 4f8cfad1444..42287ff84ce 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -116,7 +116,7 @@ You might also be interested in templates for various ### Set a default template for merge requests and issues -> `Default.md` template [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302) in GitLab 14.8. +> `Default.md` (case insensitive) template [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302) in GitLab 14.8. In a project, you can choose a default description template for new issues and merge requests. As a result, every time a new merge request or issue is created, it's pre-filled with the text you @@ -129,9 +129,9 @@ Prerequisites: To set a default description template for merge requests, either: -- [Create a merge request template](#create-a-merge-request-template) named `Default.md` or `default.md` +- [In GitLab 14.8 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302), [create a merge request template](#create-a-merge-request-template) named `Default.md` (case insensitive) and save it in `.gitlab/merge_request_templates/`. - This doesn't overwrite the default template if one has been set in the project settings. + This [doesn't overwrite](#priority-of-default-description-templates) the default template if one has been set in the project settings. - Users on GitLab Premium and higher: set the default template in project settings: 1. On the top bar, select **Menu > Projects** and find your project. @@ -142,9 +142,9 @@ To set a default description template for merge requests, either: To set a default description template for issues, either: -- [Create an issue template](#create-an-issue-template) named `Default.md` or `default.md` +- [In GitLab 14.8 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302), [create an issue template](#create-an-issue-template) named `Default.md` (case insensitive) and save it in `.gitlab/issue_templates/`. - This doesn't overwrite the default template if one has been set in the project settings. + This [doesn't overwrite](#priority-of-default-description-templates) the default template if one has been set in the project settings. - Users on GitLab Premium and higher: set the default template in project settings: 1. On the top bar, select **Menu > Projects** and find your project. @@ -159,15 +159,15 @@ headings, lists, and so on. You can also provide `issues_template` and `merge_requests_template` attributes in the [Projects REST API](../../api/projects.md) to keep your default issue and merge request templates up to date. -#### Priority of description templates +#### Priority of default description templates When you set [merge request and issue description templates](#set-a-default-template-for-merge-requests-and-issues) in various places, they have the following priorities in a project. The ones higher up override the ones below: -1. Template selected in project settings. -1. `Default.md` from the parent group. -1. `Default.md` from the project repository. +1. Template set in project settings. +1. `Default.md` (case insensitive) from the parent group. +1. `Default.md` (case insensitive) from the project repository. ## Example description template diff --git a/doc/user/project/highlighting.md b/doc/user/project/highlighting.md index ef0c787b9d3..37ec7c8e8d3 100644 --- a/doc/user/project/highlighting.md +++ b/doc/user/project/highlighting.md @@ -42,7 +42,7 @@ To override syntax highlighting for a file type: After the changes merge into your [default branch](repository/branches/default.md), all `*.pl` files in your project are highlighted in your preferred language. -You can also extend the highlighting with common gateway interface (CGI) options, such as: +You can also extend the highlighting with Common Gateway Interface (CGI) options, such as: ``` conf # JSON file with .erb in it diff --git a/doc/user/project/img/time_tracking_report_v13_12.png b/doc/user/project/img/time_tracking_report_v13_12.png Binary files differdeleted file mode 100644 index 2132ca01cf4..00000000000 --- a/doc/user/project/img/time_tracking_report_v13_12.png +++ /dev/null diff --git a/doc/user/project/img/time_tracking_report_v15_1.png b/doc/user/project/img/time_tracking_report_v15_1.png Binary files differnew file mode 100644 index 00000000000..a9ddefebb2f --- /dev/null +++ b/doc/user/project/img/time_tracking_report_v15_1.png diff --git a/doc/user/project/import/cvs.md b/doc/user/project/import/cvs.md index 17d717bdc61..c150e124ac8 100644 --- a/doc/user/project/import/cvs.md +++ b/doc/user/project/import/cvs.md @@ -71,6 +71,6 @@ Here's a few links to get you started with the migration: - [Migrate using the `cvs-fast-export` tool](https://gitlab.com/esr/cvs-fast-export) - [Stack Overflow post on importing the CVS repository](https://stackoverflow.com/a/11490134/974710) -- [Convert a CVS repository to Git](http://www.techrepublic.com/article/convert-cvs-repositories-to-git/) +- [Convert a CVS repository to Git](https://www.techrepublic.com/article/convert-cvs-repositories-to-git/) - [Man page of the `git-cvsimport` tool](https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-cvsimport.html) - [Migrate using `reposurgeon`](http://www.catb.org/~esr/reposurgeon/repository-editing.html#conversion) diff --git a/doc/user/project/import/gitlab_com.md b/doc/user/project/import/gitlab_com.md index 6913cda0cd5..4103367accc 100644 --- a/doc/user/project/import/gitlab_com.md +++ b/doc/user/project/import/gitlab_com.md @@ -19,12 +19,12 @@ you'll need to follow the instructions for [exporting a project](../settings/imp  -Go to the **Import Projects** tab, then click on **GitLab.com**, and you are redirected to GitLab.com +Go to the **Import Projects** tab, then select **GitLab.com**, and you are redirected to GitLab.com for permission to access your projects. After accepting, you are automatically redirected to the importer.  -To import a project, click "Import". The importer imports your repository and issues. +To import a project, select **Import**. The importer imports your repository and issues. Once the importer is done, a new GitLab project is created with your imported data. ## Automate group and project import **(PREMIUM)** diff --git a/doc/user/project/import/img/import_projects_from_repo_url.png b/doc/user/project/import/img/import_projects_from_repo_url.png Binary files differdeleted file mode 100644 index fd3eae98ebf..00000000000 --- a/doc/user/project/import/img/import_projects_from_repo_url.png +++ /dev/null diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index a44669e738e..8fb495cd0db 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -71,19 +71,19 @@ To import Jira issues to a GitLab project:  -1. Click the **Import from** dropdown list and select the Jira project that you wish to import issues from. +1. Select the **Import from** dropdown list and select the Jira project that you wish to import issues from. In the **Jira-GitLab user mapping template** section, the table shows to which GitLab users your Jira users are mapped. When the form appears, the dropdown list defaults to the user conducting the import. -1. To change any of the mappings, click the dropdown list in the **GitLab username** column and +1. To change any of the mappings, select the dropdown list in the **GitLab username** column and select the user you want to map to each Jira user. The dropdown list may not show all the users, so use the search bar to find a specific user in this GitLab project. -1. Click **Continue**. You're presented with a confirmation that import has started. +1. Select **Continue**. You're presented with a confirmation that import has started. While the import is running in the background, you can navigate away from the import status page to the issues page, and you can see the new issues appearing in the issues list. diff --git a/doc/user/project/import/manifest.md b/doc/user/project/import/manifest.md index 131732d2bae..f04048980e7 100644 --- a/doc/user/project/import/manifest.md +++ b/doc/user/project/import/manifest.md @@ -53,13 +53,13 @@ As a result, the following projects are created: To start the import: -1. From your GitLab dashboard click **New project**. +1. From your GitLab dashboard select **New project**. 1. Switch to the **Import project** tab. -1. Click on the **Manifest file** button. +1. Select **Manifest file**. 1. Provide GitLab with a manifest XML file. 1. Select a group you want to import to (you need to create a group first if you don't have one). -1. Click **List available repositories**. At this point, you are redirected +1. Select **List available repositories**. At this point, you are redirected to the import status page with projects list based on the manifest file. -1. Check the list and click **Import all repositories** to start the import. +1. Check the list and select **Import all repositories** to start the import.  diff --git a/doc/user/project/import/repo_by_url.md b/doc/user/project/import/repo_by_url.md index 0b96238006e..5163f957171 100644 --- a/doc/user/project/import/repo_by_url.md +++ b/doc/user/project/import/repo_by_url.md @@ -9,20 +9,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can import your existing repositories by providing the Git URL: -<!-- vale gitlab.Spelling = NO --> -<!-- vale gitlab.SubstitutionWarning = NO --> - -1. From your GitLab dashboard click **New project**. -1. Switch to the **Import project** tab. -1. Click on the **Repo by URL** button. -1. Fill in the "Git repository URL" and the remaining project fields. -1. Click **Create project** to begin the import process. -1. Once complete, you will be redirected to your newly created project. - -<!-- vale gitlab.Spelling = YES --> -<!-- vale gitlab.SubstitutionWarning = YES --> - - +1. On the top bar, select **Menu > Create new project**. +1. Select the **Import project** tab. +1. Select **Repository by URL**. +1. Enter a **Git repository URL**. +1. Complete the remaining fields. +1. Select **Create project**. + +Your newly created project is displayed. ## Automate group and project import **(PREMIUM)** diff --git a/doc/user/project/integrations/bugzilla.md b/doc/user/project/integrations/bugzilla.md index 0f7ce182e1a..ac4b9d0769b 100644 --- a/doc/user/project/integrations/bugzilla.md +++ b/doc/user/project/integrations/bugzilla.md @@ -38,7 +38,7 @@ project pages. This link takes you to the appropriate Bugzilla project. You can also disable [GitLab internal issue tracking](../issues/index.md) in this project. Learn more about the steps and consequences of disabling GitLab issues in -[Sharing and permissions](../settings/index.md#sharing-and-permissions). +[Sharing and permissions](../settings/index.md#configure-project-visibility-features-and-permissions). ## Reference Bugzilla issues in GitLab diff --git a/doc/user/project/integrations/discord_notifications.md b/doc/user/project/integrations/discord_notifications.md index b7e25b815fc..3780ea37c0b 100644 --- a/doc/user/project/integrations/discord_notifications.md +++ b/doc/user/project/integrations/discord_notifications.md @@ -32,6 +32,6 @@ With the webhook URL created in the Discord channel, you can set up the Discord 1. Ensure that the **Active** toggle is enabled. 1. Check the checkboxes corresponding to the GitLab events for which you want to send notifications to Discord. 1. Paste the webhook URL that you copied from the create Discord webhook step. -1. Configure the remaining options and click the **Save changes** button. +1. Configure the remaining options and select the **Save changes** button. The Discord channel you created the webhook for now receives notification of the GitLab events that were configured. diff --git a/doc/user/project/integrations/harbor.md b/doc/user/project/integrations/harbor.md index 2a1b12057aa..1319c9e74cd 100644 --- a/doc/user/project/integrations/harbor.md +++ b/doc/user/project/integrations/harbor.md @@ -31,7 +31,7 @@ GitLab supports integrating Harbor projects at the group or project level. Compl 1. Turn on the **Active** toggle under **Enable Integration**. 1. Provide the Harbor configuration information: - **Harbor URL**: The base URL of Harbor instance which is being linked to this GitLab project. For example, `https://harbor.example.net`. - - **Harbor project name**: The project name in the Harbor instance. For example, `testproject`. + - **Harbor project name**: The project name in the Harbor instance. For example, `testproject`. - **Username**: Your username in the Harbor instance, which should meet the requirements in [prerequisites](#prerequisites). - **Password**: Password of your username. @@ -39,7 +39,7 @@ GitLab supports integrating Harbor projects at the group or project level. Compl After the Harbor integration is activated: -- The global variables `$HARBOR_USER`, `$HARBOR_PASSWORD`, `$HARBOR_URL`, and `$HARBOR_PROJECT` are created for CI/CD use. +- The global variables `$HARBOR_USERNAME`, `$HARBOR_PASSWORD`, `$HARBOR_URL`, and `$HARBOR_PROJECT` are created for CI/CD use. - The project-level integration settings override the group-level integration settings. ## Secure your requests to the Harbor APIs diff --git a/doc/user/project/integrations/pipeline_status_emails.md b/doc/user/project/integrations/pipeline_status_emails.md index 742ab977090..c58f5a13613 100644 --- a/doc/user/project/integrations/pipeline_status_emails.md +++ b/doc/user/project/integrations/pipeline_status_emails.md @@ -21,3 +21,6 @@ To enable pipeline status emails: **Notify only broken pipelines**. 1. Select the branches to send notifications for. 1. Select **Save changes**. + +In [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89546) +and later, pipeline notifications triggered by blocked users are not delivered. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index 6e751b907eb..4c8e648537c 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -38,7 +38,7 @@ Next, the Ingress needs to be annotated for Prometheus monitoring. Two new annot - `prometheus.io/scrape: "true"` - `prometheus.io/port: "10254"` -Managing these settings depends on how NGINX Ingress has been deployed. If you have deployed via the [official Helm chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress), metrics can be enabled with `controller.stats.enabled` along with the required annotations. Alternatively it is possible to edit the NGINX Ingress YML directly in the [Kubernetes dashboard](https://github.com/kubernetes/dashboard). +Managing these settings depends on how NGINX Ingress has been deployed. If you have deployed via the [official Helm chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress), metrics can be enabled with `controller.stats.enabled` along with the required annotations. Alternatively it is possible to edit the NGINX Ingress YAML directly in the [Kubernetes dashboard](https://github.com/kubernetes/dashboard). ## Specifying the Environment label diff --git a/doc/user/project/integrations/redmine.md b/doc/user/project/integrations/redmine.md index bcab8d05f69..a989b418199 100644 --- a/doc/user/project/integrations/redmine.md +++ b/doc/user/project/integrations/redmine.md @@ -38,7 +38,7 @@ For example, this is a configuration for a project named `gitlab-ci`: You can also disable [GitLab internal issue tracking](../issues/index.md) in this project. Learn more about the steps and consequences of disabling GitLab issues in -[Sharing and permissions](../settings/index.md#sharing-and-permissions). +[Sharing and permissions](../settings/index.md#configure-project-visibility-features-and-permissions). ## Reference Redmine issues in GitLab diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 870554100b7..bd93fbcbcbc 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -91,7 +91,7 @@ the error message and keep troubleshooting from there. You might see an entry like the following in your Sidekiq log: ```plaintext -2019-01-10_13:22:08.42572 2019-01-10T13:22:08.425Z 6877 TID-abcdefg ProjectServiceWorker JID-3bade5fb3dd47a85db6d78c5 ERROR: {:class=>"ProjectServiceWorker", :service_class=>"SlackService", :message=>"SSL_connect returned=1 errno=0 state=error: certificate verify failed"} +2019-01-10_13:22:08.42572 2019-01-10T13:22:08.425Z 6877 TID-abcdefg Integrations::ExecuteWorker JID-3bade5fb3dd47a85db6d78c5 ERROR: {:class=>"Integrations::ExecuteWorker :service_class=>"SlackService", :message=>"SSL_connect returned=1 errno=0 state=error: certificate verify failed"} ``` This issue occurs when there is a problem with GitLab communicating with Slack, diff --git a/doc/user/project/integrations/webex_teams.md b/doc/user/project/integrations/webex_teams.md index dd4cdb632e6..e8b2470cf13 100644 --- a/doc/user/project/integrations/webex_teams.md +++ b/doc/user/project/integrations/webex_teams.md @@ -33,6 +33,6 @@ notifications: 1. Ensure that the **Active** toggle is enabled. 1. Select the checkboxes corresponding to the GitLab events you want to receive in Webex Teams. 1. Paste the **Webhook** URL for the Webex Teams space. -1. Configure the remaining options and then click **Test settings and save changes**. +1. Configure the remaining options and then select **Test settings and save changes**. The Webex Teams space begins to receive all applicable GitLab events. diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index 2bf6b4bbe01..ed62a34f6a3 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -1050,6 +1050,9 @@ Pipeline events are triggered when the status of a pipeline changes. In [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53159) and later, the pipeline webhook returns only the latest jobs. +In [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89546) +and later, pipeline webhooks triggered by blocked users are not processed. + Request header: ```plaintext @@ -1126,6 +1129,15 @@ Payload example: "email": "user@gitlab.com" } }, + "source_pipeline":{ + "project":{ + "id": 41, + "web_url": "https://gitlab.example.com/gitlab-org/upstream-project", + "path_with_namespace": "gitlab-org/upstream-project", + }, + "pipeline_id": 30, + "job_id": 3401 + }, "builds":[ { "id": 380, @@ -1301,6 +1313,9 @@ Job events are triggered when the status of a job changes. The `commit.id` in the payload is the ID of the pipeline, not the ID of the commit. +In [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89546) +and later, job events triggered by blocked users are not processed. + Request header: ```plaintext @@ -1662,7 +1677,7 @@ Payload example: { "id": 1, "created_at": "2020-11-02 12:55:12 UTC", - "description": "v1.0 has been released", + "description": "v1.1 has been released", "name": "v1.1", "released_at": "2020-11-02 12:55:12 UTC", "tag": "v1.1", diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index ac7d447961c..e4ce736684b 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -40,7 +40,9 @@ including: ## Group webhooks **(PREMIUM)** You can configure a group webhook, which is triggered by events -that occur across all projects in the group. +that occur across all projects in the group. If you configure identical webhooks +in a group and a project, they are both triggered by an event in the +project. Group webhooks can also be configured to listen for events that are specific to a group, including: @@ -171,7 +173,8 @@ work you must have Ruby installed. ruby print_http_body.rb 8000 ``` -1. In GitLab, add your webhook receiver as `http://my.host:8000/`. +1. In GitLab, [configure the webhook](#configure-a-webhook-in-gitlab) and add your + receiver's URL, for example, `http://receiver.example.com:8000/`. 1. Select **Test**. You should see something like this in the console: diff --git a/doc/user/project/integrations/youtrack.md b/doc/user/project/integrations/youtrack.md index 6c70a5e679b..25fc9c4e1c3 100644 --- a/doc/user/project/integrations/youtrack.md +++ b/doc/user/project/integrations/youtrack.md @@ -29,7 +29,7 @@ project pages. This link takes you to the appropriate YouTrack project. You can also disable [GitLab internal issue tracking](../issues/index.md) in this project. Learn more about the steps and consequences of disabling GitLab issues in -[Sharing and permissions](../settings/index.md#sharing-and-permissions). +[Sharing and permissions](../settings/index.md#configure-project-visibility-features-and-permissions). ## Reference YouTrack issues in GitLab diff --git a/doc/user/project/integrations/zentao.md b/doc/user/project/integrations/zentao.md index 67125c3ebbf..0256c52e4a3 100644 --- a/doc/user/project/integrations/zentao.md +++ b/doc/user/project/integrations/zentao.md @@ -10,11 +10,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w [ZenTao](https://www.zentao.net/) is a web-based project management platform. +The following versions of ZenTao are supported: + +- ZenTao 15.4 +- ZenTao Pro 10.2 +- ZenTao Biz 5.2 +- ZenTao Max 2.2 + ## Configure ZenTao -This integration requires a ZenTao API secret key. +This integration requires a ZenTao API secret key. -Complete these steps in ZenTao: +Complete these steps in ZenTao: 1. Go to your **Admin** page and select **Develop > Application**. 1. Select **Add Application**. @@ -32,7 +39,7 @@ Complete these steps in GitLab: 1. Turn on the **Active** toggle under **Enable Integration**. 1. Provide the ZenTao configuration information: - **ZenTao Web URL**: The base URL of the ZenTao instance web interface you're linking to this GitLab project (for example, `example.zentao.net`). - - **ZenTao API URL** (optional): The base URL to the ZenTao instance API. Defaults to the Web URL value if not set. + - **ZenTao API URL** (optional): The base URL to the ZenTao instance API. Defaults to the Web URL value if not set. - **ZenTao API token**: Use the key you generated when you [configured ZenTao](#configure-zentao). - **ZenTao Product ID**: To display issues from a single ZenTao product in a given GitLab project. The Product ID can be found in the ZenTao product page under **Settings > Overview**. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index d731ceb5aca..c8ecb2fd2e6 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -70,17 +70,17 @@ GitLab automatically loads the last board you visited. To create a new issue board: -1. Click the dropdown with the current board name in the upper left corner of the issue boards page. -1. Click **Create new board**. +1. Select the dropdown with the current board name in the upper left corner of the issue boards page. +1. Select **Create new board**. 1. Enter the new board's name and select its scope: milestone, labels, assignee, or weight. ### Delete an issue board To delete the currently active issue board: -1. Click the dropdown with the current board name in the upper left corner of the issue boards page. -1. Click **Delete board**. -1. Click **Delete** to confirm. +1. Select the dropdown with the current board name in the upper left corner of the issue boards page. +1. Select **Delete board**. +1. Select **Delete** to confirm. ## Issue boards use cases @@ -289,7 +289,7 @@ assignee list: Now that the assignee list is added, you can assign or unassign issues to that user by [moving issues](#move-issues-and-lists) to and from an assignee list. -To remove an assignee list, just as with a label list, click the trash icon. +To remove an assignee list, just as with a label list, select the trash icon.  @@ -307,7 +307,7 @@ milestone, giving you more freedom and visibility on the issue board. To add a m Like the assignee lists, you're able to [drag issues](#move-issues-and-lists) to and from a milestone list to manipulate the milestone of the dragged issues. -As in other list types, click the trash icon to remove a list. +As in other list types, select the trash icon to remove a list.  @@ -392,8 +392,8 @@ Examples: To set a WIP limit for a list: 1. Navigate to a Project or Group board of which you're a member. -1. Click the settings icon in a list's header. -1. Next to **Work In Progress Limit**, click **Edit**. +1. Select the settings icon in a list's header. +1. Next to **Work In Progress Limit**, select **Edit**. 1. Enter the maximum number of issues. 1. Press <kbd>Enter</kbd> to save. diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 9d23a63b940..402ce4bebec 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -31,12 +31,12 @@ There are two ways to change an issue's confidentiality. The first way is to edit the issue and toggle the confidentiality checkbox. After you save the issue, the confidentiality of the issue is updated. -The second way is to locate the Confidentiality section in the sidebar and click +The second way is to locate the **Confidentiality** section in the sidebar and select **Edit**. A popup should appear and give you the option to turn on or turn off confidentiality. | Turn off confidentiality | Turn on confidentiality | | :-----------: | :----------: | -|  |  | +|  |  | Every change from regular to confidential and vice versa, is indicated by a system note in the issue's comments. @@ -84,6 +84,9 @@ There are two kinds of level access for confidential issues. The general rule is that confidential issues are visible only to members of a project with at least the Reporter role. However, a guest user can also create confidential issues, but can only view the ones that they created themselves. +Users with the Guest role or non-members can also read the confidential issue if they are assigned to the issue. +When a Guest user or non-member is unassigned from a confidential issue, +they can no longer view it. Confidential issues are also hidden in search results for unprivileged users. For example, here's what a user with the Maintainer role and the Guest role diff --git a/doc/user/project/issues/csv_import.md b/doc/user/project/issues/csv_import.md index a3f6ee5e61e..2fe3d78194c 100644 --- a/doc/user/project/issues/csv_import.md +++ b/doc/user/project/issues/csv_import.md @@ -25,7 +25,7 @@ To import issues: 1. Go to your project's Issues list page. 1. Open the import feature, depending if the project has issues: - - Existing issues are present: Select the import icon at the top right, next to **Edit issues**. + - Existing issues are present: Select the import icon at the top right, next to **Edit issues**. - Project has no issues: Select **Import CSV** in the middle of the page. 1. Select the file you want to import, and then select **Import issues**. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index e5dde0ed451..d1b27f6eab0 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -29,7 +29,7 @@ For a video overview, see [Design Management (GitLab 12.2)](https://www.youtube. - On self-managed instances, a GitLab administrator must [enable LFS globally](../../../administration/lfs/index.md). - On both GitLab.com and self-managed instances, LFS must be - [enabled for the project itself](../settings/index.md#sharing-and-permissions). + [enabled for the project itself](../settings/index.md#configure-project-visibility-features-and-permissions). If enabled globally, LFS is enabled by default for all projects. If you have disabled it for your project, you must enable it again. @@ -92,7 +92,7 @@ The design you selected opens. You can then [zoom in](#zoom-in-on-a-design) on i When viewing a design, you can move to other designs. To do so, either: -- In the top-right corner, select **Go to previous design** (**{angle-left}**) or **Go to next design** (**{angle-right}**). +- In the top-right corner, select **Go to previous design** (**{chevron-lg-left}**) or **Go to next design** (**{chevron-lg-right}**). - Press <kbd>Left</kbd> or <kbd>Right</kbd> on your keyboard. To return to the issue view, either: diff --git a/doc/user/search/img/issue_search_by_id_v15_0.png b/doc/user/project/issues/img/issue_search_by_id_v15_0.png Binary files differindex 411cebc0ccb..411cebc0ccb 100644 --- a/doc/user/search/img/issue_search_by_id_v15_0.png +++ b/doc/user/project/issues/img/issue_search_by_id_v15_0.png diff --git a/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png b/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png Binary files differnew file mode 100644 index 00000000000..e6050aec0de --- /dev/null +++ b/doc/user/project/issues/img/turn_off_confidentiality_v15_1.png diff --git a/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png b/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png Binary files differnew file mode 100644 index 00000000000..24a7ad554f8 --- /dev/null +++ b/doc/user/project/issues/img/turn_on_confidentiality_v15_1.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index bd0cf92e320..a2dbc8581d9 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -45,7 +45,7 @@ To learn how the GitLab Strategic Marketing department uses GitLab issues with [ - [Health status](managing_issues.md#health-status) - [Cross-link issues](crosslinking_issues.md) - [Sort issue lists](sorting_issue_lists.md) -- [Search for issues](../../search/index.md#filter-issue-and-merge-request-lists) +- [Search for issues](managing_issues.md#filter-the-list-of-issues) - [Epics](../../group/epics/index.md) - [Issue boards](../issue_board.md) - [Issues API](../../../api/issues.md) diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 7db66dd013b..fbdce211295 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -225,8 +225,6 @@ When you're creating a new issue, you can complete the following fields: ## Edit an issue -> Reordering list items [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.0. - You can edit an issue's title and description. Prerequisites: @@ -239,11 +237,23 @@ To edit an issue: 1. Edit the available fields. 1. Select **Save changes**. -You can also reorder list items, which include bullet, numerical, and task list items. -To reorder list items: +### Reorder list items in the issue description + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15260) in GitLab 15.0. + +When you view an issue that has a list in the description, you can also reorder the list items. + +Prerequisites: + +- You must have at least the Reporter role for the project, be the author of the issue, or be + assigned to the issue. +- The issue's description must have an [ordered, unordered](../../markdown.md#lists), or + [task](../../markdown.md#task-lists) list. -1. Hover over the list item row to make the drag icon visible. -1. Click and hold the drag icon. +To reorder list items, when viewing an issue: + +1. Hover over the list item row to make the drag icon (**{drag-vertical}**) visible. +1. Select and hold the drag icon. 1. Drag the row to the new position in the list. 1. Release the drag icon. @@ -471,7 +481,7 @@ Referenced issues are still displayed, but are not closed automatically. The automatic issue closing is disabled by default in a project if the project has the issue tracker disabled. If you want to enable automatic issue closing, make sure to -[enable GitLab Issues](../settings/index.md#sharing-and-permissions). +[enable GitLab Issues](../settings/index.md#configure-project-visibility-features-and-permissions). Changing this setting applies only to new merge requests or commits. Already closed issues remain as they are. @@ -554,6 +564,52 @@ To add an issue to an [iteration](../../group/iterations/index.md): Alternatively, you can use the `/iteration` [quick action](../quick_actions.md#issues-merge-requests-and-epics). +## View all issues assigned to you + +To view all issues assigned to you: + +1. On the top bar, put your cursor in the **Search** box. +1. From the dropdown list, select **Issues assigned to me**. + +Or: + +- To use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>i</kbd>. +- On the top bar, on the top right, select **{issues}** **Issues**. + +## Filter the list of issues + +> - Filtering by iterations was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in GitLab 13.6. +> - Filtering by iterations was moved from GitLab Ultimate to GitLab Premium in 13.9. +> - Filtering by type was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322755) in GitLab 13.10 [with a flag](../../../administration/feature_flags.md) named `vue_issues_list`. Disabled by default. +> - Filtering by type was [enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/322755) in GitLab 14.10. +> - Filtering by type is generally available in GitLab 15.1. [Feature flag `vue_issues_list`](https://gitlab.com/gitlab-org/gitlab/-/issues/359966) removed. + +To filter the list of issues: + +1. Above the list of issues, select **Search or filter results...**. +1. In the dropdown list that appears, select the attribute you want to filter by. +1. Select or type the operator to use for filtering the attribute. The following operators are + available: + - `=`: Is + - `!=`: Is not ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18059) in GitLab 12.7) +1. Enter the text to filter the attribute by. + You can filter some attributes by **None** or **Any**. +1. Repeat this process to filter by multiple attributes. Multiple attributes are joined by a logical + `AND`. + +GitLab displays the results on-screen, but you can also +[retrieve them as an RSS feed](../../search/index.md#retrieve-search-results-as-feed). + +### Filter issues by ID + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1. + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Issues > List**. +1. In the **Search** box, type the issue ID. For example, enter filter `#10` to return only issue 10. + + + ## Copy issue reference To refer to an issue elsewhere in GitLab, you can use its full URL or a short reference, which looks like diff --git a/doc/user/project/issues/multiple_assignees_for_issues.md b/doc/user/project/issues/multiple_assignees_for_issues.md index 279f297d016..105dcd529c8 100644 --- a/doc/user/project/issues/multiple_assignees_for_issues.md +++ b/doc/user/project/issues/multiple_assignees_for_issues.md @@ -34,7 +34,7 @@ it clear that their role is complete. ## How it works From an opened issue, expand the right sidebar, locate the assignees entry, -and click on **Edit**. From the dropdown menu, select as many users as you want +and select **Edit**. From the dropdown menu, select as many users as you want to assign the issue to.  diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index b8151ac873a..028dd2ea473 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -61,7 +61,7 @@ You can also add a linked issue from a commit message or the description in anot ## Remove a linked issue -In the **Linked issues** section of an issue, click the remove button (**{close}**) on the +In the **Linked issues** section of an issue, select the remove button (**{close}**) on the right-side of each issue token to remove. Due to the bi-directional relationship, the relationship no longer appears in either issue. diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 2cc23b14857..160dade87bb 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -16,8 +16,7 @@ Labels are a key part of [issue boards](issue_board.md). With labels you can: - Categorize [epics](../group/epics/index.md), issues, and merge requests using colors and descriptive titles like `bug`, `feature request`, or `docs`. - Dynamically filter and manage [epics](../group/epics/index.md), issues, and merge requests. -- [Search lists of issues, merge requests, and epics](../search/index.md#search-issues-and-merge-requests), - as well as [issue boards](../search/index.md#issue-boards). +- Search lists of issues, merge requests, and epics, as well as issue boards. ## Types of labels @@ -125,8 +124,7 @@ To do so: 1. Select **Create project label**. 1. Fill in the name field. You can't specify a description if creating a label this way. You can add a description later by [editing the label](#edit-a-label). -1. Optional. Select a color by selecting from the available colors, or enter a hex color value for - a specific color. +1. Select a color by selecting from the available colors, or enter a hex color value for a specific color. 1. Select **Create**. ### Create a group label @@ -160,8 +158,7 @@ To do so: 1. Select **Create group label**. 1. Fill in the name field. You can't specify a description if creating a label this way. You can add a description later by [editing the label](#edit-a-label). -1. Optional. Select a color by selecting from the available colors,enter input a hex color value - for a specific color. +1. Select a color by selecting from the available colors,enter input a hex color value for a specific color. 1. Select **Create**. ## Edit a label @@ -336,7 +333,7 @@ For example, filtering by the `platform::*` label returns issues that have `plat `platform::Android`, or `platform::Linux` labels. NOTE: -Filtering by scoped labels not available on the [issues or merge requests dashboard pages](../search/index.md#search-issues-and-merge-requests). +Filtering by scoped labels not available on the issues or merge requests dashboard pages. ### Scoped labels examples diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index 28bd353d8cc..7bea57d180b 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -225,9 +225,10 @@ GitLab users can request to become a member of a project. An email is sent to the most recently active project maintainers. Up to ten project maintainers are notified. -Any project maintainer can approve or decline the request. +Any project owner or maintainer can approve or decline the request. +Project maintainers cannot approve Owner role access requests. -If a project does not have any maintainers, the notification is sent to the +If a project does not have any direct owners or maintainers, the notification is sent to the most recently active owners of the project's group. If you change your mind before your request is approved, select diff --git a/doc/user/project/merge_requests/accessibility_testing.md b/doc/user/project/merge_requests/accessibility_testing.md index 612f499bb65..b8907532066 100644 --- a/doc/user/project/merge_requests/accessibility_testing.md +++ b/doc/user/project/merge_requests/accessibility_testing.md @@ -46,10 +46,10 @@ To define the `a11y` job for GitLab 12.9 and later: ```yaml stages: - accessibility - + variables: a11y_urls: "https://about.gitlab.com https://gitlab.com/users/sign_in" - + include: - template: "Verify/Accessibility.gitlab-ci.yml" ``` diff --git a/doc/user/project/merge_requests/approvals/index.md b/doc/user/project/merge_requests/approvals/index.md index f0ab4d606ad..014936208c6 100644 --- a/doc/user/project/merge_requests/approvals/index.md +++ b/doc/user/project/merge_requests/approvals/index.md @@ -22,7 +22,8 @@ flexibility: - Specify a list of users who act as [code owners](../../code_owners.md) for specific files, and require their approval before work can merge. -You can configure merge request approvals on a per-project basis. Administrators of +You can configure merge request approvals on a per-project basis, and +[on the group level](../../../group/index.md#group-merge-request-approval-settings). Administrators of [GitLab Premium](https://about.gitlab.com/pricing/) and [GitLab Ultimate](https://about.gitlab.com/pricing/) self-managed GitLab instances can also configure approvals @@ -103,7 +104,18 @@ Without the approvals, the work cannot merge. Required approvals enable multiple to determine who should review the work. - Require an [approval before merging code that causes test coverage to decline](../../../../ci/pipelines/settings.md#coverage-check-approval-rule) - Users on GitLab Ultimate can also [require approval from a security team](../../../application_security/index.md#security-approvals-in-merge-requests) - before merging code that could introduce a vulnerability. + before merging code that could introduce a vulnerability. + +## Invalid rules + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334698) in GitLab 15.1. + +Whenever an approval rule cannot be satisfied, the rule will be displayed as `Invalid`. This applies to the following conditions: + +- The only eligible approver is the author of the merge request. +- No eligible approvers (either groups or users) have been assigned to the approval rule. + +These rules will be automatically approved to unblock their respective merge requests. ## Related topics diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index 17a42e1b540..21cf5cca4d1 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -250,6 +250,10 @@ For more information about this validation error, read You can use [scan result policies](../../../application_security/policies/scan-result-policies.md#scan-result-policy-editor) to define security approvals based on the status of vulnerabilities in the merge request and the default branch. Details for each security policy is shown in the Security Approvals section of your Merge Request configuration. +The security approval rules are applied to all merge requests until the pipeline is complete. The application of the +security approval rules prevents users from merging in code before the security scans run. Once the pipeline is +complete, the security approval rules are checked to determine if the security approvals are still required. +  These policies are both created and edited in the [security policy editor](../../../application_security/policies/index.md#policy-editor). diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index 9c2b54888fb..9295ea4c310 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -139,7 +139,7 @@ You can also enforce merge request approval settings: - At the [instance level](../../../admin_area/merge_requests_approvals.md), which apply to all groups on an instance and, therefore, all projects. -- On a [top-level group](../../../group/index.md#group-approval-settings), which apply to all subgroups +- On a [top-level group](../../../group/index.md#group-merge-request-approval-settings), which apply to all subgroups and projects. If the settings are inherited by a group or project, they cannot be changed in the group or project diff --git a/doc/user/project/merge_requests/cherry_pick_changes.md b/doc/user/project/merge_requests/cherry_pick_changes.md index fb41ec3eff6..14f3979cf34 100644 --- a/doc/user/project/merge_requests/cherry_pick_changes.md +++ b/doc/user/project/merge_requests/cherry_pick_changes.md @@ -18,7 +18,7 @@ to cherry-pick the changes introduced by that merge request.  -After you click that button, a modal displays a +After you select that button, a modal displays a [branch filter search box](../repository/branches/index.md#branch-filter-search-box) where you can choose to either: @@ -69,12 +69,12 @@ git cherry-pick -m 2 7a39eb0 You can cherry-pick merge requests from the same project, or forks of the same project, from the GitLab user interface: -1. In the merge request's secondary menu, click **Commits** to display the commit details page. -1. Click on the **Options** dropdown and select **Cherry-pick** to show the cherry-pick modal. +1. In the merge request's secondary menu, select **Commits** to display the commit details page. +1. Select the **Options** dropdown and select **Cherry-pick** to show the cherry-pick modal. 1. In **Pick into project** and **Pick into branch**, select the destination project and branch:  1. Optional. Select **Start a new merge request** if you're ready to create a merge request. -1. Click **Cherry-pick**. +1. Select **Cherry-pick**. ## Related topics diff --git a/doc/user/project/merge_requests/code_quality.md b/doc/user/project/merge_requests/code_quality.md index 7e8ef9272d4..623af914692 100644 --- a/doc/user/project/merge_requests/code_quality.md +++ b/doc/user/project/merge_requests/code_quality.md @@ -28,6 +28,20 @@ Code Quality: - Is available by using [Auto Code Quality](../../../topics/autodevops/stages.md#auto-code-quality), provided by [Auto DevOps](../../../topics/autodevops/index.md). - Can be extended through [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) or a [custom tool](#implementing-a-custom-tool). +## Summary of features per tier + +Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), +as shown in the following table: + +| Capability | In Free | In Premium | In Ultimate | +|:----------------------------------------------------------------------|:--------------------|:--------------------|:-------------------| +| [Configure scanners](#configuring-jobs-using-variables) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [Integrate custom scanners](#implementing-a-custom-tool) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [Generate JSON or HTML report artifacts](#generate-an-html-report) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [See findings in merge request widget](#code-quality-widget) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [See reports in CI pipelines](#code-quality-reports) | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | +| [See findings in merge request diff view](#code-quality-in-diff-view) | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | + ## Code Quality Widget > [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. @@ -242,7 +256,7 @@ This can be done: - For a single pipeline run: 1. Go to **CI/CD > Pipelines** - 1. Click **Run pipeline** + 1. Select **Run pipeline** 1. Add `CODE_QUALITY_DISABLED` as the variable key, with any value. ### Using with merge request pipelines @@ -402,32 +416,40 @@ CI/CD variable to `html`. This is useful if you just want to view the report in human-readable format or to publish this artifact on GitLab Pages for even easier reviewing. +To generate both JSON and HTML report files, add another job to your template by using `extends: code_quality`: + ```yaml include: - template: Code-Quality.gitlab-ci.yml -code_quality: +code_quality_html: + extends: code_quality variables: REPORT_FORMAT: html artifacts: paths: [gl-code-quality-report.html] ``` -It's also possible to generate both JSON and HTML report files by defining -another job and using `extends: code_quality`: +NOTE: +Adding a job means your code is scanned twice: once to generate a JSON report and once to generate an HTML report. + +You can also generate _only_ an HTML report instead of the standard JSON report. To do so, set `REPORT_FORMAT` to `html` in the existing job: ```yaml include: - template: Code-Quality.gitlab-ci.yml -code_quality_html: - extends: code_quality +code_quality: variables: REPORT_FORMAT: html artifacts: paths: [gl-code-quality-report.html] ``` +WARNING: +If you only generate an HTML report, you can't see your results in the [merge request widget](#code-quality-widget), [pipeline report](#code-quality-reports), or [diff view](#code-quality-in-diff-view). +These features require a JSON report. + ## Extending functionality ### Using Analysis Plugins @@ -557,9 +579,9 @@ GitLab only uses the Code Quality artifact from the latest created job (with the If multiple jobs in a pipeline generate a code quality artifact, those of earlier jobs are ignored. To avoid confusion, configure only one job to generate a `gl-code-quality-report.json`. -### Rubocop errors +### RuboCop errors -When using Code Quality jobs on a Ruby project, you can encounter problems running Rubocop. +When using Code Quality jobs on a Ruby project, you can encounter problems running RuboCop. For example, the following error can appear when using either a very recent or very old version of Ruby: @@ -569,15 +591,15 @@ Unknown Ruby version 2.7 found in `.ruby-version`. (RuboCop::ValidationError) Supported versions: 2.1, 2.2, 2.3, 2.4, 2.5 ``` -This is caused by the default version of Rubocop used by the check engine not covering +This is caused by the default version of RuboCop used by the check engine not covering support for the Ruby version in use. -To use a custom version of Rubocop that +To use a custom version of RuboCop that [supports the version of Ruby used by the project](https://docs.rubocop.org/rubocop/compatibility.html#support-matrix), you can [override the configuration through a `.codeclimate.yml` file](https://docs.codeclimate.com/docs/rubocop#using-rubocops-newer-versions) created in the project repository. -For example, to specify using Rubocop release **0.67**: +For example, to specify using RuboCop release **0.67**: ```yaml version: "2" diff --git a/doc/user/project/merge_requests/commit_templates.md b/doc/user/project/merge_requests/commit_templates.md index b9b65d759dc..6f9bc452b96 100644 --- a/doc/user/project/merge_requests/commit_templates.md +++ b/doc/user/project/merge_requests/commit_templates.md @@ -92,8 +92,8 @@ Commit message templates support these variables: Any line containing only an empty variable is removed. If the line to be removed is both preceded and followed by an empty line, the preceding empty line is also removed. -After you edit a commit message on an open merge request, GitLab will -not automatically update the commit message again. +After you edit a commit message on an open merge request, GitLab +automatically updates the commit message again. To restore the commit message to the project template, reload the page. ## Related topics diff --git a/doc/user/project/merge_requests/csv_export.md b/doc/user/project/merge_requests/csv_export.md index df527ec6ebf..aaa9bec945f 100644 --- a/doc/user/project/merge_requests/csv_export.md +++ b/doc/user/project/merge_requests/csv_export.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Exporting merge requests CSV enables you and your team to export all the data collected from merge requests into a comma-separated values (CSV) file, which stores tabular data in plain text. -To export merge requests to CSV, navigate to your **Merge requests** from the sidebar of a project and click **Export to CSV**. +To export merge requests to CSV, navigate to your **Merge requests** from the sidebar of a project and select **Export to CSV**. ## CSV Output diff --git a/doc/user/project/merge_requests/drafts.md b/doc/user/project/merge_requests/drafts.md index 637b682d603..13cc68f02dd 100644 --- a/doc/user/project/merge_requests/drafts.md +++ b/doc/user/project/merge_requests/drafts.md @@ -23,14 +23,14 @@ the **Merge** button until you remove the **Draft** flag: There are several ways to flag a merge request as a draft: -- **Viewing a merge request**: In the top right corner of the merge request, click **Mark as draft**. +- **Viewing a merge request**: In the top right corner of the merge request, select **Mark as draft**. - **Creating or editing a merge request**: Add `[Draft]`, `Draft:` or `(Draft)` to - the beginning of the merge request's title, or click **Start the title with Draft:** + the beginning of the merge request's title, or select **Start the title with Draft:** below the **Title** field. - **Commenting in an existing merge request**: Add the `/draft` [quick action](../quick_actions.md#issues-merge-requests-and-epics) in a comment. This quick action is a toggle, and can be repeated to change the status - again. This quick action discards any other text in the comment. + back to Ready. - **Creating a commit**: Add `draft:`, `Draft:`, `fixup!`, or `Fixup!` to the beginning of a commit message targeting the merge request's source branch. This is not a toggle, and adding this text again in a later commit doesn't mark the @@ -40,19 +40,18 @@ There are several ways to flag a merge request as a draft: When a merge request is ready to be merged, you can remove the `Draft` flag in several ways: -- **Viewing a merge request**: In the top right corner of the merge request, click **Mark as ready**. +- **Viewing a merge request**: In the top right corner of the merge request, select **Mark as ready**. Users with at least the Developer role - can also scroll to the bottom of the merge request description and click **Mark as ready**: + can also scroll to the bottom of the merge request description and select **Mark as ready**:  - **Editing an existing merge request**: Remove `[Draft]`, `Draft:` or `(Draft)` - from the beginning of the title, or click **Remove the Draft: prefix from the title** + from the beginning of the title, or select **Remove the Draft: prefix from the title** below the **Title** field. -- **Commenting in an existing merge request**: Add the `/draft` +- **Commenting in an existing merge request**: Add the `/ready` [quick action](../quick_actions.md#issues-merge-requests-and-epics) - in a comment in the merge request. This quick action is a toggle, and can be repeated - to change the status back. This quick action discards any other text in the comment. + in a comment in the merge request. In [GitLab 13.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/15332), when you mark a merge request as ready, notifications are triggered to @@ -64,9 +63,9 @@ When viewing or searching in your project's merge requests list, you can include draft merge requests: 1. Go to your project and select **Merge requests**. -1. In the navigation bar, click **Open**, **Merged**, **Closed**, or **All** to +1. In the navigation bar, select **Open**, **Merged**, **Closed**, or **All** to filter by merge request status. -1. Click the search box to display a list of filters and select **Draft**, or +1. Select the search box to display a list of filters and select **Draft**, or enter the word `draft`. 1. Select `=`. 1. Select **Yes** to include drafts, or **No** to exclude, and press **Return** diff --git a/doc/user/project/merge_requests/getting_started.md b/doc/user/project/merge_requests/getting_started.md index c1986a80ca0..09ee828ffd3 100644 --- a/doc/user/project/merge_requests/getting_started.md +++ b/doc/user/project/merge_requests/getting_started.md @@ -126,7 +126,7 @@ When creating a merge request, select the **Delete source branch when merge request accepted** option, and the source branch is deleted when the merge request is merged. To make this option enabled by default for all new merge requests, enable it in the -[project's settings](../settings/index.md#merge-request-settings). +[project's settings](../settings/index.md#configure-merge-request-settings-for-a-project). This option is also visible in an existing merge request next to the merge request button and can be selected or cleared before merging. @@ -140,35 +140,6 @@ is set for deletion, the merge request widget displays the  -### Branch retargeting on merge **(FREE SELF)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/320902) in GitLab 13.9. -> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/320902) in GitLab 13.9. -> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/320895) GitLab 13.10. - -In specific circumstances, GitLab can retarget the destination branch of -open merge request, if the destination branch merges while the merge request is -open. Merge requests are often chained in this manner, with one merge request -depending on another: - -- **Merge request 1**: merge `feature-alpha` into `main`. -- **Merge request 2**: merge `feature-beta` into `feature-alpha`. - -These merge requests are usually handled in one of these ways: - -- Merge request 1 is merged into `main` first. Merge request 2 is then - retargeted to `main`. -- Merge request 2 is merged into `feature-alpha`. The updated merge request 1, which - now contains the contents of `feature-alpha` and `feature-beta`, is merged into `main`. - -GitLab retargets up to four merge requests when their target branch is merged into -`main`, so you don't need to perform this operation manually. Merge requests from -forks are not retargeted. - -The feature today works only on merge. Clicking the **Remove source branch** button -after the merge request was merged will not automatically retarget a merge request. -This improvement is [tracked as a follow-up](https://gitlab.com/gitlab-org/gitlab/-/issues/321559). - ## Recommendations and best practices for merge requests - When working locally in your branch, add multiple commits and only push when diff --git a/doc/user/project/merge_requests/img/ff_merge_rebase_v14_9.png b/doc/user/project/merge_requests/img/ff_merge_rebase_v14_9.png Binary files differdeleted file mode 100644 index 17ce42e7a69..00000000000 --- a/doc/user/project/merge_requests/img/ff_merge_rebase_v14_9.png +++ /dev/null diff --git a/doc/user/search/img/filter_approved_by_merge_requests_v14_6.png b/doc/user/project/merge_requests/img/filter_approved_by_merge_requests_v14_6.png Binary files differindex 8d47fdc2652..8d47fdc2652 100644 --- a/doc/user/search/img/filter_approved_by_merge_requests_v14_6.png +++ b/doc/user/project/merge_requests/img/filter_approved_by_merge_requests_v14_6.png diff --git a/doc/user/search/img/filter_approver_merge_requests_v14_6.png b/doc/user/project/merge_requests/img/filter_approver_merge_requests_v14_6.png Binary files differindex 58950031378..58950031378 100644 --- a/doc/user/search/img/filter_approver_merge_requests_v14_6.png +++ b/doc/user/project/merge_requests/img/filter_approver_merge_requests_v14_6.png diff --git a/doc/user/search/img/filtering_merge_requests_by_date_v14_6.png b/doc/user/project/merge_requests/img/filtering_merge_requests_by_date_v14_6.png Binary files differindex 398820f7864..398820f7864 100644 --- a/doc/user/search/img/filtering_merge_requests_by_date_v14_6.png +++ b/doc/user/project/merge_requests/img/filtering_merge_requests_by_date_v14_6.png diff --git a/doc/user/search/img/filtering_merge_requests_by_environment_v14_6.png b/doc/user/project/merge_requests/img/filtering_merge_requests_by_environment_v14_6.png Binary files differindex c35f2c8a58b..c35f2c8a58b 100644 --- a/doc/user/search/img/filtering_merge_requests_by_environment_v14_6.png +++ b/doc/user/project/merge_requests/img/filtering_merge_requests_by_environment_v14_6.png diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 510dcd82907..30b69c2fff5 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -45,13 +45,100 @@ If your group contains subgroups, this view also displays merge requests from th To view all merge requests assigned to you: +<!-- vale gitlab.FirstPerson = NO --> + 1. On the top bar, put your cursor in the **Search** box. 1. From the dropdown list, select **Merge requests assigned to me**. -Or, to use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>. +<!-- vale gitlab.FirstPerson = YES --> + +Or: + +- To use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>. +- On the top bar, on the top right, select **{merge-request-open}** **Merge requests**. + Then select one of the following: + - [Review requests](reviews/index.md). + - Merge requests assigned. + +## Filter the list of merge requests + +To filter the list of merge requests: + +1. Above the list of merge requests, select **Search or filter results...**. +1. In the dropdown list that appears, select the attribute you wish to filter by. +1. Select or type the operator to use for filtering the attribute. The following operators are + available: + - `=`: Is + - `!=`: Is not ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18059) in GitLab 12.7) +1. Enter the text to filter the attribute by. + You can filter some attributes by **None** or **Any**. +1. Repeat this process to filter by multiple attributes. Multiple attributes are joined by a logical + `AND`. + +GitLab displays the results on-screen, but you can also +[retrieve them as an RSS feed](../../search/index.md#retrieve-search-results-as-feed). + +### Filter merge requests by ID + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1. + +You can filter the **Merge Request** list to find merge requests by their ID. + +For example, enter filter `#30` to return only merge request 30. + +### Filter merge requests by approvers **(PREMIUM)** + +> Moved to GitLab Premium in 13.9. + +To filter merge requests by an individual eligible approver ([Code owner](../code_owners.md)), you can type (or select from +the dropdown list) **Approver** and select the user. + + + +### Filter merge requests by "approved by" **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30335) in GitLab 13.0. +> - Moved to GitLab Premium in 13.9. + +To filter merge requests already approved by a specific individual, you can type (or select from +the dropdown list) **Approved-By** and select the user. + + -You can [search and filter](../../search/index.md#filter-issue-and-merge-request-lists), -the results, or select a merge request to begin a review. +### Filter merge requests by reviewer + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47605) in GitLab 13.7. + +To filter review requested merge requests for a specific individual, you can type (or select from +the dropdown list) **Reviewer** and select the user. + +### Filter merge requests by environment or deployment date + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44041) in GitLab 13.6. + +To filter merge requests by deployment data, such as the environment or a date, +you can type (or select from the dropdown list) the following: + +- Environment +- Deployed-before +- Deployed-after + +NOTE: +Projects using a [fast-forward merge method](methods/index.md#fast-forward-merge) +do not return results, as this method does not create a merge commit. + +When filtering by an environment, a dropdown list presents all environments that +you can choose from: + + + +When filtering by `Deployed-before` or `Deployed-after`, the date refers to when +the deployment to an environment (triggered by the merge commit) completed successfully. +You must enter the deploy date manually. Deploy dates +use the format `YYYY-MM-DD`, and must be quoted if you wish to specify +both a date and time (`"YYYY-MM-DD HH:MM"`): + + ## Add changes to a merge request @@ -84,8 +171,7 @@ a merge request, or: 1. Select **Edit**. 1. Search for the user you want to assign, and select the user. -The merge request is added to the user's -[assigned merge request list](../../search/index.md#search-issues-and-merge-requests). +The merge request is added to the user's assigned merge request list. ### Assign multiple users **(PREMIUM)** @@ -136,6 +222,35 @@ To delete a merge request: 1. Go to the merge request you want to delete, and select **Edit**. 1. Scroll to the bottom of the page, and select **Delete merge request**. +### Update merge requests when target branch merges **(FREE SELF)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/320902) in GitLab 13.9. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/320902) in GitLab 13.9. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/320895) GitLab 13.10. + +Merge requests are often chained together, with one merge request depending on +the code added or changed in another merge request. To support keeping individual +merge requests small, GitLab can update up to four open merge requests when their +target branch merges into `main`. For example: + +- **Merge request 1**: merge `feature-alpha` into `main`. +- **Merge request 2**: merge `feature-beta` into `feature-alpha`. + +If these merge requests are open at the same time, and merge request 1 (`feature-alpha`) +merges into `main`, GitLab updates the destination of merge request 2 from `feature-alpha` +to `main`. + +Merge requests with interconnected content updates are usually handled in one of these ways: + +- Merge request 1 is merged into `main` first. Merge request 2 is then + retargeted to `main`. +- Merge request 2 is merged into `feature-alpha`. The updated merge request 1, which + now contains the contents of `feature-alpha` and `feature-beta`, is merged into `main`. + +This feature works only when a merge request is merged. Selecting **Remove source branch** +after merging does not retarget open merge requests. This improvement is +[proposed as a follow-up](https://gitlab.com/gitlab-org/gitlab/-/issues/321559). + ## Request attention to a merge request > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343528) in GitLab 14.10 [with a flag](../../../administration/feature_flags.md) named `mr_attention_requests`. Disabled by default. @@ -190,7 +305,7 @@ For a software developer working in a team: 1. You checkout a new branch, and submit your changes through a merge request. 1. You gather feedback from your team. 1. You work on the implementation optimizing code with [Code Quality reports](code_quality.md). -1. You verify your changes with [Unit test reports](../../../ci/unit_test_reports.md) in GitLab CI/CD. +1. You verify your changes with [Unit test reports](../../../ci/testing/unit_test_reports.md) in GitLab CI/CD. 1. You avoid using dependencies whose license is not compatible with your project with [License Compliance reports](../../compliance/license_compliance/index.md). 1. You request the [approval](approvals/index.md) from your manager. 1. Your manager: @@ -215,7 +330,7 @@ For a web developer writing a webpage for your company's website: - [Create a merge request](creating_merge_requests.md) - [Review a merge request](reviews/index.md) - [Authorization for merge requests](authorization_for_merge_requests.md) -- [Testing and reports](testing_and_reports_in_merge_requests.md) +- [Testing and reports](../../../ci/testing/index.md) - [GitLab keyboard shortcuts](../../shortcuts.md) - [Comments and threads](../../discussions/index.md) - [Suggest code changes](reviews/suggestions.md) diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index ac1c61f2e72..9182cf11566 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -16,7 +16,7 @@ finish and remember to merge the request manually. ## How it works -When you click "Merge When Pipeline Succeeds", the status of the merge +When you select "Merge When Pipeline Succeeds", the status of the merge request is updated to show the impending merge. If you can't wait for the pipeline to succeed, you can choose **Merge immediately** in the dropdown menu on the right of the main button. @@ -56,10 +56,11 @@ As a result, [disabling GitLab CI/CD pipelines](../../../ci/enable_or_disable_ci does not disable this feature, as it is possible to use pipelines from external CI providers with this feature. To enable it, you must: -1. Navigate to your project's **Settings > General** page. -1. Expand the **Merge requests** section. -1. In the **Merge checks** subsection, select the **Pipelines must succeed** checkbox. -1. Press **Save** for the changes to take effect. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Merge requests**. +1. Under **Merge checks**, select the **Pipelines must succeed** checkbox. +1. Select **Save**. This setting also prevents merge requests from being merged if there is no pipeline. You should be careful to configure CI/CD so that pipelines run for every merge request. @@ -104,11 +105,13 @@ for details on avoiding two pipelines for a single merge request. When the **Pipelines must succeed** checkbox is checked, [skipped pipelines](../../../ci/pipelines/index.md#skip-a-pipeline) prevent merge requests from being merged. To change this behavior: -1. Navigate to your project's **Settings > General** page. -1. Expand the **Merge requests** section. -1. In the **Merge checks** subsection, ensure **Pipelines must succeed** is checked. -1. In the **Merge checks** subsection, select the **Skipped pipelines are considered successful** checkbox. -1. Press **Save** for the changes to take effect. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Merge requests**. +1. Under **Merge checks**: + - Ensure **Pipelines must succeed** is selected. + - Select the **Skipped pipelines are considered successful** checkbox. +1. Select **Save**. ## From the command line diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md index adfa5288f81..d3221162cfd 100644 --- a/doc/user/project/merge_requests/methods/index.md +++ b/doc/user/project/merge_requests/methods/index.md @@ -77,7 +77,7 @@ When a fast-forward merge is not possible, the user is given the option to rebas NOTE: Projects using the fast-forward merge strategy can't filter merge requests -[by deployment date](../../../search/index.md#filtering-merge-requests-by-environment-or-deployment-date), +[by deployment date](../index.md#filter-merge-requests-by-environment-or-deployment-date), because no merge commit is created. When you visit the merge request page with `Fast-forward merge` @@ -87,8 +87,6 @@ method selected, you can accept it **only if a fast-forward merge is possible**. ## Rebasing in (semi-)linear merge methods -> Rebasing without running a CI/CD pipeline [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) in GitLab 14.7. - In these merge methods, you can merge only when your source branch is up-to-date with the target branch: - Merge commit with semi-linear history. @@ -96,11 +94,7 @@ In these merge methods, you can merge only when your source branch is up-to-date If a fast-forward merge is not possible but a conflict-free rebase is possible, GitLab offers you the [`/rebase` quick action](../../../../topics/git/git_rebase.md#rebase-from-the-gitlab-ui), -and the ability to **Rebase** from the user interface: - - - -In [GitLab 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) and later, you can also rebase without running a CI/CD pipeline. +and the ability to select **Rebase** from the user interface. If the target branch is ahead of the source branch and a conflict-free rebase is not possible, you must rebase the source branch locally before you can do a fast-forward merge. @@ -110,6 +104,23 @@ not possible, you must rebase the source branch locally before you can do a fast Rebasing may be required before squashing, even though squashing can itself be considered equivalent to rebasing. +### Rebase without CI/CD pipeline + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) in GitLab 14.7 [with a flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. Disabled by default. + +FLAG: +On GitLab.com and self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. +The feature is not ready for production use. + +To rebase a merge request's branch without triggering a CI/CD pipeline, select +**Rebase without pipeline** from the merge request reports section. +This option is available when fast-forward merge is not possible but a conflict-free +rebase is possible. + +Rebasing without a CI/CD pipeline saves resources in projects with a semi-linear +workflow that requires frequent rebases. + ## Related topics - [Commits history](../commits.md) diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index 7b4a41f9339..8f433c13887 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -22,7 +22,7 @@ to revert the changes introduced by that merge request.  -After you click that button, a modal appears where you can choose to +After you select that button, a modal appears where you can choose to revert the changes directly into the selected branch or you can opt to create a new merge request with the revert changes. diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index eb5a54e6119..8f77ce90436 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -89,7 +89,7 @@ comment itself. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8225) in GitLab 13.10. -If you have a review in progress, you will be presented with the option to **Add to review**: +If you have a review in progress, you are presented with the option to **Add to review**:  @@ -123,9 +123,9 @@ Use [attention requests](../index.md#request-attention-to-a-merge-request) inste After a reviewer completes their [merge request reviews](../../../discussions/index.md), the author of the merge request can request a new review from the reviewer: -1. If the right sidebar in the merge request is collapsed, click the +1. If the right sidebar in the merge request is collapsed, select the **{chevron-double-lg-left}** **Expand Sidebar** icon to expand it. -1. In the **Reviewers** section, click the **Re-request a review** icon (**{redo}**) +1. In the **Reviewers** section, select the **Re-request a review** icon (**{redo}**) next to the reviewer's name. GitLab creates a new [to-do item](../../../todos.md) for the reviewer, and sends @@ -168,11 +168,11 @@ When bulk-editing merge requests in a project, you can edit the following attrib To update multiple project merge requests at the same time: 1. In a project, go to **Merge requests**. -1. Click **Edit merge requests**. A sidebar on the right-hand side of your screen appears with +1. Select **Edit merge requests**. A sidebar on the right-hand side of your screen appears with editable fields. 1. Select the checkboxes next to each merge request you want to edit. 1. Select the appropriate fields and their values from the sidebar. -1. Click **Update all**. +1. Select **Update all**. ## Bulk edit merge requests at the group level **(PREMIUM)** @@ -188,11 +188,11 @@ When bulk editing merge requests in a group, you can edit the following attribut To update multiple group merge requests at the same time: 1. In a group, go to **Merge requests**. -1. Click **Edit merge requests**. A sidebar on the right-hand side of your screen appears with +1. Select **Edit merge requests**. A sidebar on the right-hand side of your screen appears with editable fields. 1. Select the checkboxes next to each merge request you want to edit. 1. Select the appropriate fields and their values from the sidebar. -1. Click **Update all**. +1. Select **Update all**. ## Associated features diff --git a/doc/user/project/merge_requests/reviews/suggestions.md b/doc/user/project/merge_requests/reviews/suggestions.md index 8e6794bcfa7..7360b57103b 100644 --- a/doc/user/project/merge_requests/reviews/suggestions.md +++ b/doc/user/project/merge_requests/reviews/suggestions.md @@ -82,9 +82,13 @@ in four backticks instead of three: GitLab uses a default commit message when applying suggestions: `Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)` +<!-- vale gitlab.BadPlurals = NO --> + For example, consider that a user applied 3 suggestions to 2 different files, the default commit message is: **Apply 3 suggestion(s) to 2 file(s)** +<!-- vale gitlab.BadPlurals = YES --> + These commit messages can be customized to follow any guidelines you might have. To do so, expand the **Merge requests** tab within your project's **General** settings and change the **Merge suggestions** text: diff --git a/doc/user/project/merge_requests/test_coverage_visualization.md b/doc/user/project/merge_requests/test_coverage_visualization.md index 85b5bbea284..fcbd732f8ee 100644 --- a/doc/user/project/merge_requests/test_coverage_visualization.md +++ b/doc/user/project/merge_requests/test_coverage_visualization.md @@ -68,11 +68,37 @@ A single Cobertura XML file can be no more than 10MiB. For large projects, split smaller files. See [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328772) for more details. When submitting many files, it can take a few minutes for coverage to show on a merge request. +The visualization only displays after the pipeline is complete. If the pipeline has +a [blocking manual job](../../../ci/jobs/job_control.md#types-of-manual-jobs), the +pipeline waits for the manual job before continuing and is not considered complete. +The visualization cannot be displayed if the blocking manual job did not run. + ### Artifact expiration By default, the [pipeline artifact](../../../ci/pipelines/pipeline_artifacts.md#storage) used to draw the visualization on the merge request expires **one week** after creation. +### Coverage report from child pipeline + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363301) in GitLab 15.1 [with a flag](../../../administration/feature_flags.md). Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `ci_child_pipeline_coverage_reports`. +On GitLab.com, this feature is not available. +The feature is not ready for production use. + +If the test coverage is created in jobs that are in a child pipeline, the parent pipeline must use +`strategy: depend`. + +```yaml +child_test_pipeline: + trigger: + include: + - local: path/to/child_pipeline.yml + - template: Security/SAST.gitlab-ci.yml + strategy: depend +``` + ### Automatic class path correction > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217664) in GitLab 13.8. @@ -255,7 +281,7 @@ run tests: - coverage run -m pytest - coverage report - coverage xml - coverage: '/TOTAL.*\s([.\d]+)%/' + coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: reports: coverage_report: @@ -389,3 +415,27 @@ run tests: coverage_format: cobertura path: coverage/coverage.xml ``` + +## Troubleshooting + +### Test coverage visualization not displayed + +If the test coverage visualization is not displayed in the diff view, you can check +the coverage report itself and verify that: + +- The file you are viewing in the diff view is mentioned in the coverage report. +- The `source` and `filename` nodes in the report follows the [expected structure](#automatic-class-path-correction) + to match the files in your repository. + +Report artifacts are not downloadable by default. If you want the report to be downloadable +from the job details page, add your coverage report to the artifact `paths`: + +```yaml +artifacts: + paths: + - coverage/cobertura-coverage.xml + reports: + coverage_report: + coverage_format: cobertura + path: coverage/cobertura-coverage.xml +``` diff --git a/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md index 741ac325cca..6c0086bc429 100644 --- a/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md +++ b/doc/user/project/merge_requests/testing_and_reports_in_merge_requests.md @@ -1,39 +1,11 @@ --- -stage: Verify -group: Pipeline Insights -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -description: "Test your code and display reports in merge requests" +redirect_to: '../../../ci/testing/index.md' +remove_date: '2022-08-31' --- -# Tests and reports in merge requests **(FREE)** +This document was moved to [another location](../../../ci/testing/index.md). -GitLab has the ability to test the changes included in a feature branch and display reports -or link to useful information directly from merge requests: - -| Feature | Description | -|----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Accessibility Testing](accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests. | -| [Browser Performance Testing](browser_performance_testing.md) | Quickly determine the browser performance impact of pending code changes. | -| [Load Performance Testing](load_performance_testing.md) | Quickly determine the server performance impact of pending code changes. | -| [Code Quality](code_quality.md) | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | -| [Display arbitrary job artifacts](../../../ci/yaml/index.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../../../ci/pipelines/job_artifacts.md) in merge requests. | -| [GitLab CI/CD](../../../ci/index.md) | Build, test, and deploy your code in a per-branch basis with built-in CI/CD. | -| [Unit test reports](../../../ci/unit_test_reports.md) | Configure your CI jobs to use Unit test reports, and let GitLab display a report on the merge request so that it's easier and faster to identify the failure without having to check the entire job log. | -| [License Compliance](../../compliance/license_compliance/index.md) | Manage the licenses of your dependencies. | -| [Metrics Reports](../../../ci/metrics_reports.md) | Display the Metrics Report on the merge request so that it's fast and easy to identify changes to important metrics. | -| [Multi-Project pipelines](../../../ci/pipelines/multi_project_pipelines.md) | When you set up GitLab CI/CD across multiple projects, you can visualize the entire pipeline, including all cross-project interdependencies. | -| [Merge request pipelines](../../../ci/pipelines/merge_request_pipelines.md) | Customize a specific pipeline structure for merge requests in order to speed the cycle up by running only important jobs. | -| [Pipeline Graphs](../../../ci/pipelines/index.md#visualize-pipelines) | View the status of pipelines within the merge request, including the deployment process. | -| [Test Coverage visualization](test_coverage_visualization.md) | See test coverage results for merge requests, within the file diff. | - -## Security Reports **(ULTIMATE)** - -In addition to the reports listed above, GitLab can do many types of [Security reports](../../application_security/index.md), -generated by scanning and reporting any vulnerabilities found in your project: - -| Feature | Description | -|-----------------------------------------------------------------------------------------|------------------------------------------------------------------| -| [Container Scanning](../../application_security/container_scanning/index.md) | Analyze your Docker images for known vulnerabilities. | -| [Dynamic Application Security Testing (DAST)](../../application_security/dast/index.md) | Analyze your running web applications for known vulnerabilities. | -| [Dependency Scanning](../../application_security/dependency_scanning/index.md) | Analyze your dependencies for known vulnerabilities. | -| [Static Application Security Testing (SAST)](../../application_security/sast/index.md) | Analyze your source code for known vulnerabilities. | +<!-- This redirect file can be deleted after <2022-08-31>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/project/milestones/burndown_and_burnup_charts.md b/doc/user/project/milestones/burndown_and_burnup_charts.md index 05cc424e5ee..d6fcd9fbb16 100644 --- a/doc/user/project/milestones/burndown_and_burnup_charts.md +++ b/doc/user/project/milestones/burndown_and_burnup_charts.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w [Burndown](#burndown-charts) and [burnup](#burnup-charts) charts show the progress of completing a milestone. - + ## Burndown charts @@ -19,7 +19,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Burndown charts show the number of issues over the course of a milestone. - + At a glance, you see the current state for the completion a given milestone. Without them, you would have to organize the data from the milestone and plot it @@ -106,7 +106,7 @@ Reopened issues are considered as having been opened on the day after they were Burnup charts show the assigned and completed work for a milestone. - + To view a project's burnup chart: diff --git a/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_6.png b/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_6.png Binary files differdeleted file mode 100644 index 8d6ba1d4fa7..00000000000 --- a/doc/user/project/milestones/img/burndown_and_burnup_charts_v13_6.png +++ /dev/null diff --git a/doc/user/project/milestones/img/burndown_and_burnup_charts_v15_1.png b/doc/user/project/milestones/img/burndown_and_burnup_charts_v15_1.png Binary files differnew file mode 100644 index 00000000000..58c0ddf892f --- /dev/null +++ b/doc/user/project/milestones/img/burndown_and_burnup_charts_v15_1.png diff --git a/doc/user/project/milestones/img/burndown_chart_v13_6.png b/doc/user/project/milestones/img/burndown_chart_v13_6.png Binary files differdeleted file mode 100644 index e06b24f9907..00000000000 --- a/doc/user/project/milestones/img/burndown_chart_v13_6.png +++ /dev/null diff --git a/doc/user/project/milestones/img/burndown_chart_v15_1.png b/doc/user/project/milestones/img/burndown_chart_v15_1.png Binary files differnew file mode 100644 index 00000000000..2953380292d --- /dev/null +++ b/doc/user/project/milestones/img/burndown_chart_v15_1.png diff --git a/doc/user/project/milestones/img/burnup_chart_v13_6.png b/doc/user/project/milestones/img/burnup_chart_v13_6.png Binary files differdeleted file mode 100644 index a850caba348..00000000000 --- a/doc/user/project/milestones/img/burnup_chart_v13_6.png +++ /dev/null diff --git a/doc/user/project/milestones/img/burnup_chart_v15_1.png b/doc/user/project/milestones/img/burnup_chart_v15_1.png Binary files differnew file mode 100644 index 00000000000..e89b76344ed --- /dev/null +++ b/doc/user/project/milestones/img/burnup_chart_v15_1.png diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index c2b85a2183c..b0f3179961d 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -11,17 +11,9 @@ Milestones in GitLab are a way to track issues and merge requests created to ach Milestones allow you to organize issues and merge requests into a cohesive group, with an optional start date and an optional due date. -## Milestones as Agile sprints - -Milestones can be used as Agile sprints so that you can track all issues and merge requests related to a particular sprint. To do so: - -1. Set the milestone start date and due date to represent the start and end of your Agile sprint. -1. Set the milestone title to the name of your Agile sprint, such as `November 2018 sprint`. -1. Add an issue to your Agile sprint by associating the desired milestone from the issue's right-hand sidebar. - ## Milestones as releases -Similarly, milestones can be used as releases. To do so: +Milestones can be used to track releases. To do so: 1. Set the milestone due date to represent the release date of your release and leave the milestone start date blank. 1. Set the milestone title to the version of your release, such as `Version 9.4`. @@ -117,19 +109,19 @@ Every issue and merge request can be assigned a milestone. The milestones are vi ### Filtering in list pages -From the project and group issue/merge request list pages, you can [filter](../../search/index.md#search-issues-and-merge-requests) by both group and project milestones. +From the project and group issue/merge request list pages, you can filter by both group and project milestones. ### Filtering in issue boards From [project issue boards](../issue_board.md), you can filter by both group milestones and project milestones in: -- [Search and filter bar](../../search/index.md#issue-boards) +- [Search and filter bar](../issue_board.md#filter-issues) - [Issue board configuration](../issue_board.md#configurable-issue-boards) From [group issue boards](../issue_board.md#group-issue-boards), you can filter by only group milestones in: -- [Search and filter bar](../../search/index.md#issue-boards) +- [Search and filter bar](../issue_board.md#filter-issues) - [Issue board configuration](../issue_board.md#configurable-issue-boards) ### Special milestone filters @@ -164,7 +156,7 @@ There are also tabs below these that show the following: The milestone view contains a [burndown and burnup chart](burndown_and_burnup_charts.md), showing the progress of completing a milestone. - + ### Milestone sidebar diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 5433e02b210..6daf671a751 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -53,7 +53,7 @@ search the web for `how to add dns record on <my hosting service>`. ## `A` record -A DNS A record maps a host to an IPv4 IP address. +A DNS `A` record maps a host to an IPv4 IP address. It points a root domain as `example.com` to the host's IP address as `192.192.192.192`. @@ -61,10 +61,10 @@ Example: - `example.com` => `A` => `192.192.192.192` -## CNAME record +## `CNAME` record -CNAME records define an alias for canonical name for your server (one defined -by an A record). It points a subdomain to another domain. +`CNAME` records define an alias for canonical name for your server (one defined +by an `A` record). It points a subdomain to another domain. Example: @@ -84,14 +84,14 @@ Example: Then you can register emails for `users@mail.example.com`. -## TXT record +## `TXT` record A `TXT` record can associate arbitrary text with a host or other name. A common use is for site verification. Example: -- `example.com`=> TXT => `"google-site-verification=6P08Ow5E-8Q0m6vQ7FMAqAYIDprkVV8fUf_7hZ4Qvc8"` +- `example.com`=> `TXT` => `"google-site-verification=6P08Ow5E-8Q0m6vQ7FMAqAYIDprkVV8fUf_7hZ4Qvc8"` This way, you can verify the ownership for that domain name. @@ -102,4 +102,4 @@ You can have one DNS record or more than one combined: - `example.com` => `A` => `192.192.192.192` - `www` => `CNAME` => `example.com` - `MX` => `mail.example.com` -- `example.com`=> TXT => `"google-site-verification=6P08Ow5E-8Q0m6vQ7FMAqAYIDprkVV8fUf_7hZ4Qvc8"` +- `example.com`=> `TXT` => `"google-site-verification=6P08Ow5E-8Q0m6vQ7FMAqAYIDprkVV8fUf_7hZ4Qvc8"` diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index d970c0f9ef4..2c668e2c409 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -47,13 +47,13 @@ this document for an [overview on DNS records](dns_concepts.md). #### 1. Add a custom domain to Pages -Navigate to your project's **Setting > Pages** and click **+ New domain** +Navigate to your project's **Setting > Pages** and select **+ New domain** to add your custom domain to GitLab Pages. You can choose whether to: - Add an [SSL/TLS certificate](#adding-an-ssltls-certificate-to-pages). - Leave it blank (it can be added later). -Click **Create New Domain**. +Select **Create New Domain**.  @@ -82,20 +82,20 @@ Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214718) for de Root domains (`example.com`) require: -- A [DNS A record](dns_concepts.md#a-record) pointing your domain to the Pages server. -- A [TXT record](dns_concepts.md#txt-record) to verify your domain's ownership. +- A [DNS `A` record](dns_concepts.md#a-record) pointing your domain to the Pages server. +- A [`TXT` record](dns_concepts.md#txt-record) to verify your domain's ownership. | From | DNS Record | To | | --------------------------------------------- | ---------- | --------------- | -| `example.com` | A | `35.185.44.232` | -| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `example.com` | `A` | `35.185.44.232` | +| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | For projects on GitLab.com, this IP is `35.185.44.232`. For projects living in other GitLab instances (CE or EE), please contact your sysadmin asking for this information (which IP address is Pages server running on your instance). - + WARNING: Note that if you use your root domain for your GitLab Pages website @@ -111,7 +111,7 @@ as it most likely doesn't work if you set an [`MX` record](dns_concepts.md#mx-re Subdomains (`subdomain.example.com`) require: - A DNS [`ALIAS` or `CNAME` record](dns_concepts.md#cname-record) pointing your subdomain to the Pages server. -- A DNS [TXT record](dns_concepts.md#txt-record) to verify your domain's ownership. +- A DNS [`TXT` record](dns_concepts.md#txt-record) to verify your domain's ownership. | From | DNS Record | To | |:--------------------------------------------------------|:----------------|:----------------------| @@ -122,7 +122,7 @@ Note that, whether it's a user or a project website, the DNS record should point to your Pages domain (`namespace.gitlab.io`), without any `/project-name`. - + ##### For both root and subdomains @@ -137,11 +137,11 @@ They require: | From | DNS Record | To | | ------------------------------------------------- | ---------- | ---------------------- | -| `example.com` | A | `35.185.44.232` | -| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `example.com` | `A` | `35.185.44.232` | +| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | |---------------------------------------------------+------------+------------------------| -| `www.example.com` | CNAME | `namespace.gitlab.io` | -| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `www.example.com` | `CNAME` | `namespace.gitlab.io` | +| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | If you're using Cloudflare, check [Redirecting `www.domain.com` to `domain.com` with Cloudflare](#redirecting-wwwdomaincom-to-domaincom-with-cloudflare). @@ -162,8 +162,8 @@ If you're using Cloudflare, check Once you have added all the DNS records: 1. Go back at your project's **Settings > Pages**. -1. Locate your domain name and click **Details**. -1. Click the **Retry verification** button to activate your new domain. +1. Locate your domain name and select **Details**. +1. Select the **Retry verification** button to activate your new domain.  @@ -208,15 +208,15 @@ For a root domain: | From | DNS Record | To | | ------------------------------------------------- | ---------- | ---------------------- | -| `example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | For a subdomain: | From | DNS Record | To | | ------------------------------------------------- | ---------- | ---------------------- | -| `www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | ### Adding more domain aliases @@ -241,10 +241,10 @@ can use the following setup: 1. In GitLab, verify your domain. 1. In Cloudflare, create a DNS `CNAME` record pointing `www` to `domain.com`. 1. In Cloudflare, add a Page Rule pointing `www.domain.com` to `domain.com`: - - Navigate to your domain's dashboard and click **Page Rules** + - Navigate to your domain's dashboard and select **Page Rules** on the top nav. - - Click **Create Page Rule**. - - Enter the domain `www.domain.com` and click **+ Add a Setting**. + - Select **Create Page Rule**. + - Enter the domain `www.domain.com` and select **+ Add a Setting**. - From the dropdown menu, choose **Forwarding URL**, then select the status code **301 - Permanent Redirect**. - Enter the destination URL `https://domain.com`. @@ -285,7 +285,7 @@ meet these requirements. - To add the certificate at the time you add a new domain, go to your project's **Settings > Pages > New Domain**, add the domain name and the certificate. - To add the certificate to a domain previously added, go to your - project's **Settings > Pages**, locate your domain name, click **Details** and **Edit** to add the certificate. + project's **Settings > Pages**, locate your domain name, select **Details** and **Edit** to add the certificate.  diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index cb22a200514..184e4f345c1 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -43,13 +43,13 @@ For **self-managed** GitLab instances, make sure your administrator has Once you've met the requirements, enable Let's Encrypt integration: 1. Navigate to your project's **Settings > Pages**. -1. Find your domain and click **Details**. -1. Click **Edit** in the top-right corner. +1. Find your domain and select **Details**. +1. Select **Edit** in the top-right corner. 1. Enable Let's Encrypt integration by switching **Automatic certificate management using Let's Encrypt**:  -1. Click **Save changes**. +1. Select **Save changes**. Once enabled, GitLab obtains a LE certificate and add it to the associated Pages domain. GitLab also renews it automatically. @@ -70,8 +70,8 @@ associated Pages domain. GitLab also renews it automatically. If you get an error **Something went wrong while obtaining the Let's Encrypt certificate**, first, make sure that your pages site is set to "Everyone" in your project's **Settings > General > Visibility**. This allows the Let's Encrypt Servers reach your pages site. Once this is confirmed, you can try obtaining the certificate again by following these steps: 1. Go to your project's **Settings > Pages**. -1. Click **Edit** on your domain. -1. Click **Retry**. +1. Select **Edit** on your domain. +1. Select **Retry**. 1. If you're still seeing the same error: 1. Make sure you have properly set only one `CNAME` or `A` DNS record for your domain. 1. Make sure your domain **doesn't have** an `AAAA` DNS record. @@ -86,7 +86,7 @@ Another possible cause of this error is the `_redirects` file because the curren If you've enabled Let's Encrypt integration, but a certificate is absent after an hour and you see the message, "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later.", try to remove and add the domain for GitLab Pages again by following these steps: 1. Go to your project's **Settings > Pages**. -1. Click **Remove** on your domain. +1. Select **Remove** on your domain. 1. [Add the domain again and verify it](index.md#1-add-a-custom-domain-to-pages). 1. [Enable Let's Encrypt integration for your domain](#enabling-lets-encrypt-integration-for-your-custom-domain). 1. If you still see the same message after some time: diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md index 0c848a24dec..b080bee85aa 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md @@ -23,7 +23,7 @@ highly recommendable. Let's start with an introduction to the importance of HTTPS. -## Why should I care about HTTPS? +## Why should you care about HTTPS? This might be your first question. If our sites are hosted by GitLab Pages, they are static, hence we are not dealing with server-side scripts @@ -42,7 +42,7 @@ Now we have a different picture. [According to Josh Aas](https://letsencrypt.org > _We've since come to realize that HTTPS is important for almost all websites. It's important for any website that allows people to log in with a password, any website that [tracks its users](https://www.washingtonpost.com/news/the-switch/wp/2013/12/10/nsa-uses-google-cookies-to-pinpoint-targets-for-hacking/) in any way, any website that [doesn't want its content altered](https://arstechnica.com/tech-policy/2014/09/why-comcasts-javascript-ad-injections-threaten-security-net-neutrality/), and for any site that offers content people might not want others to know they are consuming. We've also learned that any site not secured by HTTPS [can be used to attack other sites](https://krebsonsecurity.com/2015/04/dont-be-fodder-for-chinas-great-cannon/)._ Therefore, the reason why certificates are so important is that they encrypt -the connection between the **client** (you, me, your visitors) +the connection between the **client** (you, your visitors) and the **server** (where you site lives), through a keychain of authentications and validations. diff --git a/doc/user/project/pages/getting_started/pages_new_project_template.md b/doc/user/project/pages/getting_started/pages_new_project_template.md index b32d71a4887..92baba6b9c6 100644 --- a/doc/user/project/pages/getting_started/pages_new_project_template.md +++ b/doc/user/project/pages/getting_started/pages_new_project_template.md @@ -12,15 +12,15 @@ You can create a new project from a template and run the CI/CD pipeline to gener Use a template when you want to test GitLab Pages or start a new project that's already configured to generate a Pages site. -1. From the top navigation, click the **+** button and select **New project**. +1. From the top navigation, select the **+** button and select **New project**. 1. Select **Create from Template**. -1. Next to one of the templates starting with **Pages**, click **Use template**. +1. Next to one of the templates starting with **Pages**, select **Use template**.  -1. Complete the form and click **Create project**. +1. Complete the form and select **Create project**. 1. From the left sidebar, navigate to your project's **CI/CD > Pipelines** - and click **Run pipeline** to trigger GitLab CI/CD to build and deploy your + and select **Run pipeline** to trigger GitLab CI/CD to build and deploy your site. When the pipeline is finished, go to **Settings > Pages** to find the link to diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 5846ceeb1b6..1ea7500273e 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -71,7 +71,7 @@ To restrict access to your website, enable [GitLab Pages Access Control](pages_a If you ever feel the need to purge your Pages content, you can do so by going to your project's settings through the gear icon in the top right, and then -navigating to **Pages**. Click the **Remove pages** button to delete your Pages +navigating to **Pages**. Select the **Remove pages** button to delete your Pages website.  @@ -259,20 +259,20 @@ for both the `/data` and `/data/` URL paths. ## Frequently Asked Questions -### Can I download my generated pages? +### Can you download your generated pages? Sure. All you need to do is download the artifacts archive from the job page. -### Can I use GitLab Pages if my project is private? +### Can you use GitLab Pages if your project is private? Yes. GitLab Pages doesn't care whether you set your project's visibility level to private, internal or public. -### Can I create a personal or a group website +### Can you create a personal or a group website? Yes. See the documentation about [GitLab Pages domain names, URLs, and base URLs](getting_started_part_one.md). -### Do I need to create a user/group website before creating a project website? +### Do you need to create a user/group website before creating a project website? No, you don't. You can create your project first and access it under `http(s)://namespace.example.io/projectname`. diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index 9b747e04973..eb897c176fa 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -10,9 +10,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can enable Pages access control on your project if your administrator has [enabled the access control feature](../../../administration/pages/index.md#access-control) -on your GitLab instance. When enabled, only +on your GitLab instance. When enabled, only authenticated [members of your project](../../permissions.md#project-members-permissions) -(at least Guest) can access your website: +(at least Guest) can access your website, by default: <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> For a demonstration, see [Pages access controls](https://www.youtube.com/watch?v=tSPAr5mQYc8). @@ -36,7 +36,7 @@ For a demonstration, see [Pages access controls](https://www.youtube.com/watch?v - **Only project members**: Only project members are able to browse the website. - **Everyone with access**: Everyone, both logged into and logged out of GitLab, is able to browse the website, no matter their project membership. -1. Click **Save changes**. Note that your changes may not take effect immediately. GitLab Pages uses +1. Select **Save changes**. Note that your changes may not take effect immediately. GitLab Pages uses a caching mechanism for efficiency. Your changes may not take effect until that cache is invalidated, which usually takes less than a minute. diff --git a/doc/user/project/pages/redirects.md b/doc/user/project/pages/redirects.md index 1db404f4888..791b6a1550a 100644 --- a/doc/user/project/pages/redirects.md +++ b/doc/user/project/pages/redirects.md @@ -164,7 +164,7 @@ Splats also match empty strings, so the previous rule redirects ### Rewrite all requests to a root `index.html` NOTE: -If you are using [GitLab Pages integration with Let’s Encrypt](custom_domains_ssl_tls_certification/lets_encrypt_integration.md), +If you are using [GitLab Pages integration with Let's Encrypt](custom_domains_ssl_tls_certification/lets_encrypt_integration.md), you must enable it before adding this rule. Otherwise, the redirection breaks the Let's Encrypt integration. For more details, see [GitLab Pages issue 649](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/649). diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index 56f5a2d24ff..3e40a7962ae 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -21,10 +21,12 @@ When a branch is protected, the default behavior enforces these restrictions on | Protect a branch | At least the Maintainer role. | | Push to the branch | GitLab administrators and anyone with **Allowed** permission. (1) | | Force push to the branch | No one. | -| Delete the branch | No one. | +| Delete the branch | No one. (2) | 1. Users with the Developer role can create a project in a group, but might not be allowed to initially push to the [default branch](repository/branches/default.md). +1. No one can delete a protected branch using Git commands, however, users with at least Maintainer + role can [delete a protected branch from the UI or API](#delete-a-protected-branch). ### Set the default branch protection level diff --git a/doc/user/project/protected_tags.md b/doc/user/project/protected_tags.md index 5df34fcdcc4..870c544cf4c 100644 --- a/doc/user/project/protected_tags.md +++ b/doc/user/project/protected_tags.md @@ -31,7 +31,7 @@ To protect a tag, you must have at least the Maintainer role. 1. Go to the project's **Settings > Repository**. -1. From the **Tag** dropdown list, select the tag you want to protect or type and click **Create wildcard**. In the screenshot below, we chose to protect all tags matching `v*`: +1. From the **Tag** dropdown list, select the tag you want to protect or type and select **Create wildcard**. In the screenshot below, we chose to protect all tags matching `v*`:  @@ -86,6 +86,26 @@ To prevent this problem: Users can still create branches, but not tags, with the protected names. +## Delete a protected tag + +You can manually delete protected tags with the GitLab API, or the +GitLab user interface. + +Prerequisite: + +- You must have at least the Maintainer role in your project. + +To do this: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Repository > Tags**. +1. Next to the tag you want to delete, select **Delete** (**{remove}**). +1. On the confirmation dialog, enter the tag name and select **Yes, delete protected tag**. + +Protected tags can only be deleted by using GitLab either from the UI or API. +These protections prevent you from accidentally deleting a tag through local +Git commands or third-party Git clients. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index b73b381ffcd..d5a7058d3d2 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -67,7 +67,7 @@ threads. Some quick actions might not be available to all subscription tiers. | `/copy_metadata <#issue>` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Copy labels and milestone from another issue in the project. | | `/create_merge_request <branch name>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Create a new merge request starting from the current issue. | | `/done` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mark to do as done. | -| `/draft` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Toggle the draft status. | +| `/draft` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Toggle the [draft status](merge_requests/drafts.md). | | `/due <date>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set due date. Examples of valid `<date>` include `in 2 days`, `this Friday` and `December 31st`. | | `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue and mark as a duplicate of another issue. Also, mark both as related. | | `/epic <epic>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add to epic `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. | @@ -85,6 +85,7 @@ threads. Some quick actions might not be available to all subscription tiers. | `/promote_to_incident` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Promote issue to incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296787) in GitLab 14.5). | | `/page <policy name>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Start escalations for the incident ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79977) in GitLab 14.9). | | `/publish` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Publish issue to an associated [Status Page](../../operations/incident_management/status_page.md) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30906) in GitLab 13.0) | +| `/ready` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Set the [ready status](merge_requests/drafts.md#mark-merge-requests-as-ready) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90361) in GitLab 15.1). | | `/reassign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Replace current assignees with those specified. | | `/reassign_reviewer @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Replace current reviewers with those specified. | | `/rebase` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Rebase source branch. This schedules a background task that attempts to rebase the changes in the source branch on the latest commit of the target branch. If `/rebase` is used, `/merge` is ignored to avoid a race condition where the source branch is merged or deleted before it is rebased. If there are merge conflicts, GitLab displays a message that a rebase cannot be scheduled. Rebase failures are displayed with the merge request status. | diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index c0a6fa9c301..d5ddc0468e1 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -63,7 +63,7 @@ switch between ascending or descending order, select **Sort order**. You can create a release: -- [Using a job in your CI/CD pipeline](#create-a-release-by-using-a-cicd-job). +- [Using a job in your CI/CD pipeline](#creating-a-release-by-using-a-cicd-job). - [In the Releases page](#create-a-release-in-the-releases-page). - [In the Tags page](#create-a-release-in-the-tags-page). - Using the [Releases API](../../../api/releases/index.md#create-a-release). @@ -118,100 +118,71 @@ To edit release notes of an existing Git tag: You can use Markdown and drag and drop files to this text box. 1. Select **Save changes**. -### Create a release by using a CI/CD job +### Creating a release by using a CI/CD job -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19298) in GitLab 12.7. - -You can create a release directly as part of the GitLab CI/CD pipeline -by using the [`release` keyword](../../../ci/yaml/index.md#release) in the job definition. +You can create a release directly as part of the GitLab CI/CD pipeline by using the +[`release` keyword](../../../ci/yaml/index.md#release) in the job definition. The release is created only if the job processes without error. If the API returns an error during release creation, the release job fails. -#### CI/CD example of the `release` keyword +Methods for creating a release using a CI/CD job include: + +- Create a release when a Git tag is created. +- Create a release when a commit is merged to the default branch. + +#### Create a release when a Git tag is created -To create a release when you push a Git tag, or when you add a Git tag -in the UI by going to **Repository > Tags**: +In this CI/CD example, pushing a Git tag to the repository, or creating a Git tag in the UI triggers +the release. You can use this method if you prefer to create the Git tag manually, and create a +release as a result. NOTE: -Do not provide **Release notes** when you create the Git tag in the UI. -Providing release notes creates a release, resulting in the pipeline failing. +Do not provide Release notes when you create the Git tag in the UI. Providing release notes +creates a release, resulting in the pipeline failing. + +Key points in the following _extract_ of an example `.gitlab-ci.yml` file: + +- The `rules` stanza defines when the job is added to the pipeline. +- The Git tag is used in the release's name and description. ```yaml release_job: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest rules: - - if: $CI_COMMIT_TAG # Run this job when a tag is created manually + - if: $CI_COMMIT_TAG # Run this job when a tag is created script: - echo "running release_job" - release: - name: 'Release $CI_COMMIT_TAG' - description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION must be defined - tag_name: '$CI_COMMIT_TAG' # elsewhere in the pipeline. - ref: '$CI_COMMIT_TAG' - milestones: - - 'm1' - - 'm2' - - 'm3' - released_at: '2020-07-15T08:00:00Z' # Optional, is auto generated if not defined, or can use a variable. - assets: # Optional, multiple asset links - links: - - name: 'asset1' - url: 'https://example.com/assets/1' - - name: 'asset2' - url: 'https://example.com/assets/2' - filepath: '/pretty/url/1' # optional - link_type: 'other' # optional + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: '$CI_COMMIT_TAG' + description: '$CI_COMMIT_TAG' ``` -To create a release automatically when commits are pushed or merged to the default branch, -using a new Git tag that is defined with variables: +#### Create a release when a commit is merged to the default branch -```yaml -prepare_job: - stage: prepare # This stage must run before the release stage - rules: - - if: $CI_COMMIT_TAG - when: never # Do not run this job when a tag is created manually - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch - script: - - echo "EXTRA_DESCRIPTION=some message" >> variables.env # Generate the EXTRA_DESCRIPTION and TAG environment variables - - echo "TAG=v$(cat VERSION)" >> variables.env # and append to the variables.env file - artifacts: - reports: - dotenv: variables.env # Use artifacts:reports:dotenv to expose the variables to other jobs +In this CI/CD example, merging a commit to the default branch triggers the pipeline. You can use +this method if your release workflow does not create a tag manually. +Key points in the following _extract_ of an example `.gitlab-ci.yml` file: + +- The Git tag, description, and reference are created automatically in the pipeline. +- If you manually create a tag, the `release_job` job does not run. + +```yaml release_job: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest - needs: - - job: prepare_job - artifacts: true rules: - if: $CI_COMMIT_TAG when: never # Do not run this job when a tag is created manually - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch script: - echo "running release_job for $TAG" - release: - name: 'Release $TAG' - description: 'Created using the release-cli $EXTRA_DESCRIPTION' # $EXTRA_DESCRIPTION and the $TAG - tag_name: '$TAG' # variables must be defined elsewhere - ref: '$CI_COMMIT_SHA' # in the pipeline. For example, in the - milestones: # prepare_job - - 'm1' - - 'm2' - - 'm3' - released_at: '2020-07-15T08:00:00Z' # Optional, is auto generated if not defined, or can use a variable. - assets: - links: - - name: 'asset1' - url: 'https://example.com/assets/1' - - name: 'asset2' - url: 'https://example.com/assets/2' - filepath: '/pretty/url/1' # optional - link_type: 'other' # optional + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: 'v0.$CI_PIPELINE_IID' # The version is incremented per pipeline. + description: 'v0.$CI_PIPELINE_IID' + ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. ``` NOTE: @@ -219,6 +190,36 @@ Environment variables set in `before_script` or `script` are not available for e in the same job. Read more about [potentially making variables available for expanding](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6400). +#### Skip multiple pipelines when creating a release + +Creating a release using a CI/CD job could potentially trigger multiple pipelines if the associated tag does not exist already. To understand how this might happen, consider the following workflows: + +- Tag first, release second: + 1. A tag is created via UI or pushed. + 1. A tag pipeline is triggered, and runs `release` job. + 1. A release is created. + +- Release first, tag second: + 1. A pipeline is triggered when commits are pushed or merged to default branch. The pipeline runs `release` job. + 1. A release is created. + 1. A tag is created. + 1. A tag pipeline is triggered. The pipeline also runs `release` job. + +In the second workflow, the `release` job runs in multiple pipelines. To prevent this, you can use the [`workflow:rules` keyword](../../../ci/yaml/index.md#workflowrules) to determine if a release job should run in a tag pipeline: + +```yaml +release_job: + rules: + - if: $CI_COMMIT_TAG + when: never # Do not run this job in a tag pipeline + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch + script: + - echo "Create release" + release: + name: 'My awesome release' + tag_name: '$CI_COMMIT_TAG' +``` + ### Use a custom SSL CA certificate authority You can use the `ADDITIONAL_CA_CERT_BUNDLE` CI/CD variable to configure a custom SSL CA certificate authority, diff --git a/doc/user/project/repository/branches/default.md b/doc/user/project/repository/branches/default.md index f9fd1a48b9a..e087ed6c439 100644 --- a/doc/user/project/repository/branches/default.md +++ b/doc/user/project/repository/branches/default.md @@ -252,7 +252,7 @@ We are tracking this problem in [issue 20474](https://gitlab.com/gitlab-org/gitl This issue often occurs when a branch named `HEAD` is present in the repository. To fix the problem: -1. In your local repository, create a new, temporary branch and push it: +1. In your local repository, create a new temporary branch and push it: ```shell git checkout -b tmp_default && git push -u origin tmp_default diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index 351daaaca3b..6da2e5fc7ee 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -50,7 +50,7 @@ To compare branches in a repository: 1. Select **Repository > Compare** in the sidebar. 1. Select the target repository to compare with the [repository filter search box](#repository-filter-search-box). 1. Select branches to compare using the [branch filter search box](#branch-filter-search-box). -1. Click **Compare** to view the changes inline: +1. Select **Compare** to view the changes inline:  @@ -98,7 +98,7 @@ Sometimes when you have hundreds of branches you may want a more flexible matchi  -The Swap revisions feature allows you to swap the Source and Target revisions. When the Swap revisions button is clicked, the selected revisions for Source and Target will be swapped. +The Swap revisions feature allows you to swap the Source and Target revisions. When the Swap revisions button is clicked, the selected revisions for Source and Target is swapped.  diff --git a/doc/user/project/repository/csv.md b/doc/user/project/repository/csv.md index 4bf6c7451d5..27424268d2b 100644 --- a/doc/user/project/repository/csv.md +++ b/doc/user/project/repository/csv.md @@ -13,7 +13,7 @@ A comma-separated values (CSV) file is a delimited text file that uses a comma t Each line of the file is a data record. Each record consists of one or more fields, separated by commas. The use of the comma as a field separator is the source of the name for this file format. A CSV file typically stores tabular data (numbers and text) in plain text, in which case each line -will have the same number of fields. +has the same number of fields. The CSV file format is not fully standardized. Other characters can be used as column delimiters. Fields may or may not be surrounded to escape special characters. diff --git a/doc/user/project/repository/git_blame.md b/doc/user/project/repository/git_blame.md index 198993e21f3..0026834ae83 100644 --- a/doc/user/project/repository/git_blame.md +++ b/doc/user/project/repository/git_blame.md @@ -27,7 +27,7 @@ are shown. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/19299) in GitLab 12.7. -To see earlier revisions of a specific line, click **View blame prior to this change** +To see earlier revisions of a specific line, select **View blame prior to this change** until you've found the changes you're interested in viewing:  diff --git a/doc/user/project/repository/jupyter_notebooks/index.md b/doc/user/project/repository/jupyter_notebooks/index.md index d013d2802bf..1ed6dbd3348 100644 --- a/doc/user/project/repository/jupyter_notebooks/index.md +++ b/doc/user/project/repository/jupyter_notebooks/index.md @@ -28,15 +28,11 @@ GitLab. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/6589) in GitLab 14.5 as an [Alpha](../../../../policy/alpha-beta-support.md#alpha-features) release [with a flag](../../../../administration/feature_flags.md) named `jupyter_clean_diffs`. Enabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75500) in GitLab 14.9. Feature flag `jupyter_clean_diffs` removed. > - [Reintroduced toggle](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85079) in GitLab 15.0 [with a flag](../../../../administration/feature_flags.md) named `ipynb_semantic_diff`. Enabled by default. -> - Selecting between raw and cleaner diffs [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85203) in GitLab 15.0 [with a flag](../../../../administration/feature_flags.md) named `rendered_diffs_viewer`. Enabled by default. FLAG: On self-managed GitLab, by default semantic diffs are available. To hide the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md) named `ipynb_semantic_diff`. On GitLab.com, this feature is available. -FLAG: -On self-managed GitLab, by default the ability to switch between raw and rendered diffs is available. To hide the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md)named `rendered_diffs_viewer`. On GitLab.com, this feature is available. - When commits include changes to Jupyter Notebook files, GitLab: - Transforms the machine-readable `.ipynb` file into a human-readable Markdown file. diff --git a/doc/user/project/repository/managing_large_repositories.md b/doc/user/project/repository/managing_large_repositories.md index d6f88697c54..76f66f41297 100644 --- a/doc/user/project/repository/managing_large_repositories.md +++ b/doc/user/project/repository/managing_large_repositories.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments description: "Documentation on large repositories." @@ -16,13 +16,13 @@ On this page we detail several best practices to improve performance with these It's *strongly* recommended in any Git system that binary or blob files (for example, packages, audio, video, graphics, etc.) are stored as Large File Storage (LFS) objects. In such setup, the Objects are stored elsewhere, such as in Object Storage, and this can reduce the repository size significantly, thus improving performance. -Refer to the [Git LFS docs for more info](../../../topics/git/lfs/index.md). +Refer to the [Git LFS docs for more information](../../../topics/git/lfs/index.md). ## Gitaly Pack Objects Cache Gitaly, the service that provides storage for Git repositories, can be configured to cache a short rolling window of Git fetch responses. This is recommended for large repositories as it can notably reduce server load when your server receives lots of fetch traffic. -Refer to the [Gitaly Pack Objects Cache for more info](../../../administration/gitaly/configure_gitaly.md#pack-objects-cache). +Refer to the [Gitaly Pack Objects Cache for more information](../../../administration/gitaly/configure_gitaly.md#pack-objects-cache). ## Reference Architectures @@ -34,7 +34,7 @@ In these types of setups it's recommended that the GitLab environment used match Gitaly Cluster can notably improve large repository performance as it holds multiple replicas of the repository across several nodes. As a result, Gitaly Cluster can load balance read requests against those repositories and is also fault tolerant. -It's recommended for large repositories, however, Gitaly Cluster is a large solution with additional complexity of setup and management. Refer to the [Gitaly Cluster docs for more info](../../../administration/gitaly/index.md), specifically the [Before deploying Gitaly Cluster](../../../administration/gitaly/index.md#before-deploying-gitaly-cluster) section. +It's recommended for large repositories, however, Gitaly Cluster is a large solution with additional complexity of setup and management. Refer to the [Gitaly Cluster docs for more information](../../../administration/gitaly/index.md), specifically the [Before deploying Gitaly Cluster](../../../administration/gitaly/index.md#before-deploying-gitaly-cluster) section. ## Keep GitLab up to date @@ -48,4 +48,4 @@ CI/CD loads tend to be concurrent as pipelines are scheduled during set times. A When designing CI/CD pipelines, it's advisable to reduce their concurrency by staggering them to run at different times, for example, a set running at one time and then another set running several minutes later. -There's several other actions that can be explored to improve CI/CD performance with large repositories. Refer to the [Runner docs for more info](../../../ci/large_repositories/index.md). +There's several other actions that can be explored to improve CI/CD performance with large repositories. Refer to the [Runner docs for more information](../../../ci/large_repositories/index.md). diff --git a/doc/user/project/repository/mirror/index.md b/doc/user/project/repository/mirror/index.md index 1645cf7244e..fe1c9653cfe 100644 --- a/doc/user/project/repository/mirror/index.md +++ b/doc/user/project/repository/mirror/index.md @@ -167,7 +167,7 @@ for you to check: - [GitHub](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints) - [GitLab.com](../../../gitlab_com/index.md#ssh-host-keys-fingerprints) - [Launchpad](https://help.launchpad.net/SSHFingerprints) -- [Savannah](http://savannah.gnu.org/maintenance/SshAccess/) +- [Savannah](https://savannah.gnu.org/maintenance/SshAccess/) - [SourceForge](https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/) Other providers vary. You can securely gather key fingerprints with the following diff --git a/doc/user/project/repository/mirror/pull.md b/doc/user/project/repository/mirror/pull.md index 73103a9af3d..3599faf4de6 100644 --- a/doc/user/project/repository/mirror/pull.md +++ b/doc/user/project/repository/mirror/pull.md @@ -96,9 +96,9 @@ assigned when you set up pull mirroring. > Moved to GitLab Premium in 13.9. Pull mirroring uses polling to detect new branches and commits added upstream, -often minutes afterwards. If you notify GitLab by -[API](../../../../api/projects.md#start-the-pull-mirroring-process-for-a-project), -updates are pulled immediately. +often minutes afterwards. You can notify GitLab using an +[API call](../../../../api/projects.md#start-the-pull-mirroring-process-for-a-project), +but the [minimum interval for pull mirroring limits](index.md#force-an-update) is still enforced. For more information, read [Start the pull mirroring process for a project](../../../../api/projects.md#start-the-pull-mirroring-process-for-a-project). diff --git a/doc/user/project/repository/push_rules.md b/doc/user/project/repository/push_rules.md index 994cf78d98a..592aff85434 100644 --- a/doc/user/project/repository/push_rules.md +++ b/doc/user/project/repository/push_rules.md @@ -26,6 +26,9 @@ For custom push rules use [server hooks](../../../administration/server_hooks.md You can create push rules for all new projects to inherit, but they can be overridden at the project level or the [group level](../../group/index.md#group-push-rules). +All projects created after you configure global push rules inherit this +configuration. However, each existing project must be updated manually, using the +process described in [Override global push rules per project](#override-global-push-rules-per-project). Prerequisite: @@ -42,7 +45,8 @@ To create global push rules: ## Override global push rules per project The push rule of an individual project overrides the global push rule. -To override global push rules for a specific project: +To override global push rules for a specific project, or to update the rules +for an existing project to match new global push rules: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Repository**. @@ -77,10 +81,12 @@ Use these rules for your commit messages. ## Validate branch names To validate your branch names, enter a regular expression for **Branch name**. -To allow any branch name, leave empty. Your -[default branch](branches/default.md) is always allowed. +To allow any branch name, leave empty. Your [default branch](branches/default.md) +is always allowed. Certain formats of branch names are restricted by default for +security purposes. Names with 40 hexadecimal characters, similar to Git commit hashes, +are prohibited. -Examples: +Some validation examples: - Branches must start with `JIRA-`. @@ -101,11 +107,6 @@ Examples: `^[a-z0-9\\-]{4,15}$` ``` -NOTE: -In GitLab 12.10 and later, certain formats of branch names are restricted by -default for security purposes. 40-character hexadecimal names, similar to Git -commit hashes, are prohibited. - ## Prevent unintended consequences Use these rules to prevent unintended consequences. @@ -134,6 +135,8 @@ Never commit secrets, such as credential files and SSH private keys, to a versio system. In GitLab, you can use a predefined list of files to block those files from a repository. Merge requests that contain a file that matches the list are blocked. This push rule does not restrict files already committed to the repository. +You must update the configuration of existing projects to use the rule, using the +process described in [Override global push rules per project](#override-global-push-rules-per-project). Files blocked by this rule are listed below. For a complete list of criteria, refer to [`files_denylist.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/checks/files_denylist.yml). diff --git a/doc/user/project/repository/web_editor.md b/doc/user/project/repository/web_editor.md index 747bd690911..370a349b982 100644 --- a/doc/user/project/repository/web_editor.md +++ b/doc/user/project/repository/web_editor.md @@ -14,7 +14,7 @@ dropdown menu. ## Create a file -From a project's files page, click the '+' button to the right of the branch selector. +From a project's files page, select the '+' button to the right of the branch selector. Choose **New file** from the dropdown.  @@ -24,7 +24,7 @@ defaults to the branch you were viewing in the file browser. If you enter a new branch name, a checkbox displays, allowing you to start a new merge request after you commit the changes. -When you are satisfied with your new file, click **Commit Changes** at the bottom. +When you are satisfied with your new file, select **Commit Changes** at the bottom.  @@ -72,7 +72,7 @@ You don't need to construct these lines manually. Instead, you can: 1. Hover over the number of a line you want to be highlighted when sharing. 1. Right-click the number with your mouse. -1. Click **Copy Link Address** in the context menu. +1. Select **Copy Link Address** in the context menu.  @@ -82,7 +82,7 @@ The ability to create a file is great when the content is text. However, this doesn't work well for binary data such as images, PDFs, or other binary file types. In this case, you need to upload a file. -From a project's files page, click the '+' button to the right of the branch +From a project's files page, select the '+' button to the right of the branch selector. Choose **Upload file** from the dropdown:  @@ -91,7 +91,7 @@ After the upload dialog pops up, there are two ways to upload your file. Either drag and drop a file on the popup or use the **click to upload** link. After you select a file to upload, a file preview displays. -Enter a commit message, choose a branch, and click **Upload file** when you are +Enter a commit message, choose a branch, and select **Upload file** when you are ready.  @@ -101,13 +101,13 @@ ready. To keep files in the repository organized it is often helpful to create a new directory. -From a project's files page, click the plus button (`+`) to the right of the branch selector. +From a project's files page, select the plus button (`+`) to the right of the branch selector. Choose **New directory** from the dropdown.  In the new directory dialog, enter a directory name, a commit message, and choose -the target branch. Click **Create directory** to finish. +the target branch. Select **Create directory** to finish.  @@ -153,7 +153,7 @@ The branch name is based on an internal ID, and the issue title. The example screenshot above creates a branch named `2-make-static-site-auto-deploy-and-serve`. -When you click the **Create branch** button in an empty +When you select the **Create branch** button in an empty repository project, GitLab performs these actions: - Creates a default branch. diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 19c3218137f..17e55b7aac2 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -146,7 +146,7 @@ You can set description templates at various levels: - A specific [group or subgroup](description_templates.md#set-group-level-description-templates). - A specific [project](description_templates.md#set-a-default-template-for-merge-requests-and-issues). -The templates are inherited. For example, in a project, you can also access templates set for the instance or the project’s parent groups. +The templates are inherited. For example, in a project, you can also access templates set for the instance or the project's parent groups. To use a custom description template with Service Desk: diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 9b37e147a52..4eeb7c5ba83 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -4,10 +4,13 @@ group: Import info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" --- -# Project import/export **(FREE)** +# Migrating projects using file exports **(FREE)** Existing projects on any self-managed GitLab instance or GitLab.com can be exported to a file and -then imported into a new GitLab instance. +then imported into a new GitLab instance. You can also: + +- [Migrate groups](../../group/import/index.md) using the preferred method. +- [Migrate groups using file exports](../../group/settings/import_export.md). ## Set up project import/export @@ -34,8 +37,8 @@ Before you can import a project, you must export it. Prerequisites: -- Review the list of [data that will be exported](#items-that-are-exported). - Not all data is exported. +- Review the list of [items that are exported](#items-that-are-exported). + Not all items are exported. - You must have at least the Maintainer role for the project. To export a project and its data, follow these steps: @@ -74,7 +77,7 @@ The following items are **not** exported: - [Child pipeline history](https://gitlab.com/gitlab-org/gitlab/-/issues/221088) - Build traces and artifacts -- Container registry images +- Package and container registry images - CI/CD variables - Pipeline triggers - Webhooks @@ -152,6 +155,9 @@ Administrators can set the maximum import file size one of two ways: The default is `0` (unlimited). +For the GitLab.com setting, see the [Account and limit settings](../../gitlab_com/index.md#account-and-limit-settings) +section of the GitLab.com settings page. + ## Map users for import Imported users can be mapped by their public email addresses on self-managed instances, if an administrator (not an owner) does the import. diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 1e63472763d..46a6c1a049e 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -7,32 +7,29 @@ type: reference, index, howto # Project settings **(FREE)** -The **Settings** page in GitLab provides a centralized home for your -[project](../index.md) configuration options. To access it, go to your project's homepage -and, in the left navigation menu, select **Settings**. To reduce complexity, settings are -grouped by topic into sections. To display all settings in a section, select **Expand**. +Use the **Settings** page to manage the configuration options in your [project](../index.md). -In GitLab versions [13.10 and later](https://gitlab.com/groups/gitlab-org/-/epics/4842), -GitLab displays a search box to help you find the settings you want to view. +## View project settings -NOTE: -Only users who have the Maintainer role for the project and administrators can -access project settings. - -## General settings +You must have at least the Maintainer role to view project settings. -Under a project's general settings, you can find everything concerning the -functionality of a project. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. To display all settings in a section, select **Expand**. +1. Optional. Use the search box to find a setting. -### General project settings +## Edit project name and description -Adjust your project's name, description, avatar, [default branch](../repository/branches/default.md), and topics: +Use the project general settings to edit your project details. -The project description also partially supports [standard Markdown](../../markdown.md#features-extended-from-standard-markdown). -You can use [emphasis](../../markdown.md#emphasis), [links](../../markdown.md#links), and -[line-breaks](../../markdown.md#line-breaks) to add more context to the project description. +1. Sign in to GitLab with at least the Maintainer role. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. In the **Project name** text box, enter your project name. +1. In the **Project description** text box, enter your project description. +1. Under **Project avatar**, to change your project avatar, select **Choose file**. -#### Topics +## Assign topics to a project Use topics to categorize projects and find similar new projects. @@ -40,21 +37,10 @@ To assign topics to a project: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings** > **General**. -1. Under **Topics**, enter the project topics. Existing popular topics are suggested as you type. +1. In the **Topics** text box, enter the project topics. Popular topics are suggested as you type. 1. Select **Save changes**. -For GitLab.com, explore popular topics on the [Explore topics page](../working_with_projects.md#explore-topics). -When you select a topic, you can see relevant projects. - -NOTE: -The assigned topics are visible only to everyone with access to the project, -but everyone can see which topics exist at all on the GitLab instance. -Do not include sensitive information in the name of a topic. - -If you're an instance administrator, see also -[Administering topics](../../admin_area/index.md#administering-topics). - -#### Compliance frameworks **(PREMIUM)** +## Compliance frameworks **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276221) in GitLab 13.9. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/287779) in GitLab 13.12. @@ -80,7 +66,7 @@ Creating compliance frameworks on subgroups with GraphQL causes the framework to created on the root ancestor if the user has the correct permissions. The GitLab UI presents a read-only view to discourage this behavior. -#### Compliance pipeline configuration **(ULTIMATE)** +### Compliance pipeline configuration **(ULTIMATE)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3156) in GitLab 13.9, disabled behind `ff_evaluate_group_level_compliance_pipeline` [feature flag](../../../administration/feature_flags.md). > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/300324) in GitLab 13.11. @@ -186,7 +172,7 @@ as we have not [unified the user experience for these two features](https://gitl For details on the similarities and differences between these features, see [Enforce scan execution](../../application_security/#enforce-scan-execution). -##### Ensure compliance jobs are always run +### Ensure compliance jobs are always run Compliance pipelines use GitLab CI/CD to give you an incredible amount of flexibility for defining any sort of compliance jobs you like. Depending on your goals, these jobs @@ -218,7 +204,7 @@ cannot change them: This ensures that your job uses the settings you intend and that they are not overridden by project-level pipelines. -##### Avoid parent and child pipelines in GitLab 14.7 and earlier +### Avoid parent and child pipelines in GitLab 14.7 and earlier NOTE: This advice does not apply to GitLab 14.8 and later because [a fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78878) added @@ -236,25 +222,27 @@ Therefore, in projects with compliance frameworks, we recommend replacing This alternative ensures the compliance pipeline does not re-start the parent pipeline. -### Sharing and permissions +## Configure project visibility, features, and permissions -For your repository, you can set up features such as public access, repository features, -documentation, access permissions, and more. To do so from your project, -go to **Settings** > **General**, and expand the **Visibility, project features, permissions** -section. +To configure visibility, features, and permissions for a project: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility, project features, permissions** section. +1. To change the project visibility, select the dropdown list. If you select to **Public**, you limit access to some features to **Only Project Members**. +1. To allow users to request access to the project, select the **Users can request access** checkbox. +1. Use the [toggles](#project-feature-settings) to enable or disable features in the project. +1. Select **Save changes**. -You can now change the [Project visibility](../../public_access.md). -If you set **Project Visibility** to public, you can limit access to some features -to **Only Project Members**. In addition, you can select the option to -[Allow users to request access](../members/index.md#request-access-to-a-project). +### Project feature settings -Use the switches to enable or disable the following features: +Use the toggles to enable or disable features in the project. | Option | More access limit options | Description | |:---------------------------------|:--------------------------|:--------------| | **Issues** | ✓ | Activates the GitLab issues tracker. | | **Repository** | ✓ | Enables [repository](../repository/) functionality | -| **Merge requests** | ✓ | Enables [merge request](../merge_requests/) functionality; also see [Merge request settings](#merge-request-settings). | +| **Merge requests** | ✓ | Enables [merge request](../merge_requests/) functionality; also see [Merge request settings](#configure-merge-request-settings-for-a-project). | | **Forks** | ✓ | Enables [forking](../repository/forking_workflow.md) functionality. | | **Git Large File Storage (LFS)** | | Enables the use of [large files](../../../topics/git/lfs/index.md#git-large-file-storage-lfs). | | **Packages** | | Supports configuration of a [package registry](../../../administration/packages/index.md#gitlab-package-registry-administration) functionality. | @@ -269,33 +257,28 @@ Use the switches to enable or disable the following features: | **Operations** | ✓ | Control access to Operations-related features, including [Operations Dashboard](../../../operations/index.md), [Environments and Deployments](../../../ci/environments/index.md), [Feature Flags](../../../operations/feature_flags.md). | | **Metrics Dashboard** | ✓ | Control access to [metrics dashboard](../integrations/prometheus.md). | -Some features depend on others: +When you disable a feature, the following additional features are also disabled: -- If you disable the **Issues** option, GitLab also removes the following - features: +- If you disable the **Issues** feature, project users cannot use: - **Issue Boards** - - [**Service Desk**](#service-desk) + - **Service Desk** + - Project users can still access **Milestones** from merge requests. - NOTE: - When the **Issues** option is disabled, you can still access **Milestones** - from merge requests. - -- Additionally, if you disable both **Issues** and **Merge Requests**, you cannot access: +- If you disable **Issues** and **Merge Requests**, project users cannot use: - **Labels** - **Milestones** -- If you disable **Repository** functionality, GitLab also disables the following - features for your project: +- If you disable **Repository**, project users cannot access: - **Merge requests** - **CI/CD** - **Container Registry** - **Git Large File Storage** - **Packages** -- Metrics dashboard access requires reading both project environments and deployments. +- Metrics dashboard access requires reading project environments and deployments. Users with access to the metrics dashboard can also access environments and deployments. -#### Disabling the CVE ID request button **(FREE SAAS)** +## Disabling the CVE ID request button **(FREE SAAS)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41203) in GitLab 13.4, only for public projects on GitLab.com. @@ -305,12 +288,12 @@ setting **Enable CVE ID requests in the issue sidebar**.  -#### Disabling email notifications +## Disabling email notifications Project owners can disable all [email notifications](../../profile/notifications.md) related to the project by selecting the **Disable email notifications** checkbox. -### Merge request settings +## Configure merge request settings for a project Set up your project's merge request settings: @@ -326,20 +309,20 @@ Set up your project's merge request settings: - Configure [merge and squash commit message templates](../merge_requests/commit_templates.md). - Configure [the default target project](../merge_requests/creating_merge_requests.md#set-the-default-target-project) for merge requests coming from forks. -### Service Desk +## Service Desk Enable [Service Desk](../service_desk.md) for your project to offer customer support. -### Export project +## Export project Learn how to [export a project](import_export.md#import-a-project-and-its-data) in GitLab. -### Advanced settings +## Advanced settings Here you can run housekeeping, archive, rename, transfer, [remove a fork relationship](#removing-a-fork-relationship), or delete a project. -#### Archiving a project +## Archiving a project Archiving a project makes it read-only for all users and indicates that it's no longer actively maintained. Projects that have been archived can also be @@ -353,11 +336,11 @@ in project listings. To archive a project: 1. Navigate to your project's **Settings > General**. -1. Under **Advanced**, click **Expand**. -1. In the **Archive project** section, click the **Archive project** button. +1. Under **Advanced**, select **Expand**. +1. In the **Archive project** section, select **Archive project**. 1. Confirm the action when asked to. -#### Unarchiving a project +## Unarchiving a project Unarchiving a project removes the read-only restriction on a project, and makes it available in project listings. Only project owners and administrators have the @@ -373,21 +356,21 @@ To find an archived project: 1. Select **Explore projects**. 1. In the **Sort projects** dropdown box, select **Show archived projects**. 1. In the **Filter by name** field, provide the project's name. - 1. Click the link to the project to open its **Details** page. + 1. Select the link to the project to open its **Details** page. Next, to unarchive the project: 1. Navigate to your project's **Settings > General**. -1. Under **Advanced**, click **Expand**. -1. In the **Unarchive project** section, click the **Unarchive project** button. +1. Under **Advanced**, select **Expand**. +1. In the **Unarchive project** section, select **Unarchive project**. 1. Confirm the action when asked to. -#### Renaming a repository +## Renaming a repository NOTE: Only project maintainers and administrators have the [permissions](../../permissions.md#project-members-permissions) to rename a repository. Not to be confused with a project's name where it can also be -changed from the [general project settings](#general-project-settings). +changed from the [general project settings](#edit-project-name-and-description). A project's repository name defines its URL (the one you use to access the project via a browser) and its place on the file disk where GitLab is installed. @@ -395,15 +378,15 @@ project via a browser) and its place on the file disk where GitLab is installed. To rename a repository: 1. Navigate to your project's **Settings > General**. -1. Under **Advanced**, click **Expand**. +1. Under **Advanced**, select **Expand**. 1. Under **Change path**, update the repository's path. -1. Click **Change path**. +1. Select **Change path**. Remember that this can have unintended side effects since everyone with the old URL can't push or pull. Read more about what happens with the [redirects when renaming repositories](../repository/index.md#what-happens-when-a-repository-path-changes). -#### Transferring an existing project into another namespace +## Transferring an existing project into another namespace NOTE: Only project owners and administrators have the [permissions](../../permissions.md#project-members-permissions) @@ -440,7 +423,7 @@ NOTE: GitLab administrators can use the [administration interface](../../admin_area/index.md#administering-projects) to move any project to any namespace if needed. -##### Transferring a GitLab.com project to a different subscription tier +## Transferring a GitLab.com project to a different subscription tier When you transfer a project from a namespace that's licensed for GitLab SaaS Premium or Ultimate to Free, some data related to the paid features is deleted. @@ -448,7 +431,7 @@ For example, [project access tokens](../../../user/project/settings/project_acce [pipeline subscriptions](../../../ci/pipelines/multi_project_pipelines.md#trigger-a-pipeline-when-an-upstream-project-is-rebuilt) and [test cases](../../../ci/test_cases/index.md) are deleted. -#### Delete a project +## Delete a project You can mark a project to be deleted. @@ -470,18 +453,20 @@ WARNING: The default deletion behavior for projects was changed to [delayed project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6, and then to [immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. -#### Delayed project deletion **(PREMIUM)** +### Delayed project deletion **(PREMIUM)** + +> [Enabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89466) in GitLab 15.1. -Projects in a group (not a personal namespace) can be deleted after a delay period. Multiple settings can affect whether +Projects can be deleted after a delay period. Multiple settings can affect whether delayed project deletion is enabled for a particular project: -- Self-managed instance [settings](../../admin_area/settings/visibility_and_access_controls.md#default-delayed-project-deletion). +- Self-managed instance [settings](../../admin_area/settings/visibility_and_access_controls.md#deletion-protection). You can enable delayed project deletion as the default setting for new groups, and configure the number of days for the delay. For GitLab.com, see the [GitLab.com settings](../../gitlab_com/index.md#delayed-project-deletion). - Group [settings](../../group/index.md#enable-delayed-project-deletion) to enabled delayed project deletion for all projects in the group. -##### Delete a project immediately +### Delete a project immediately > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/191367) in GitLab 14.1. @@ -505,16 +490,16 @@ The following are deleted: - Your project and its repository. - All related resources including issues and merge requests. -#### Restore a project **(PREMIUM)** +## Restore a project **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6. To restore a project marked for deletion: 1. Navigate to your project, and select **Settings > General > Advanced**. -1. In the Restore project section, click the **Restore project** button. +1. In the Restore project section, select **Restore project**. -#### Removing a fork relationship +## Removing a fork relationship Forking is a great way to [contribute to a project](../repository/forking_workflow.md) of which you're not a member. @@ -529,7 +514,7 @@ To restore the fork relationship, [use the API](../../../api/projects.md#create- To do so: 1. Navigate to your project's **Settings > General > Advanced**. -1. Under **Remove fork relationship**, click the likewise-labeled button. +1. Under **Remove fork relationship**, select the likewise-labeled button. 1. Confirm the action by typing the project's path as instructed. NOTE: diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index e332b74f908..77a53874777 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -36,13 +36,15 @@ You can use project access tokens: - Consider [disabling project access tokens](#enable-or-disable-project-access-token-creation) to lower potential abuse. -You cannot use project access tokens to create other access tokens. +You cannot use project access tokens to create other group, project, or personal access tokens. Project access tokens inherit the [default prefix setting](../../admin_area/settings/account_and_limit_settings.md#personal-access-token-prefix) configured for personal access tokens. ## Create a project access token +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89114) in GitLab 15.1, Owners can select Owner role for project access tokens. + To create a project access token: 1. On the top bar, select **Menu > Projects** and find your project. @@ -116,6 +118,8 @@ Bot users for projects: - Are included in a project's member list but cannot be modified. - Cannot be added to any other project. +- Can have a maximum role of Owner for a project. For more information, see + [Create a project access token](../../../api/project_access_tokens.md#create-a-project-access-token). When the project access token is [revoked](#revoke-a-project-access-token): diff --git a/doc/user/project/time_tracking.md b/doc/user/project/time_tracking.md index 0891e02f3f7..971ecf66a3c 100644 --- a/doc/user/project/time_tracking.md +++ b/doc/user/project/time_tracking.md @@ -102,7 +102,7 @@ type `/spend 1h 2021-01-31`. If you type a future date, no time is logged. -### Remove time spent +### Subtract time spent Prerequisites: @@ -112,11 +112,10 @@ To subtract time, enter a negative value. For example, `/spend -3d` removes thre days from the total time spent. You can't go below 0 minutes of time spent, so if you remove more time than already entered, GitLab ignores the subtraction. -To remove all the time spent at once, use the `/remove_time_spent` [quick action](quick_actions.md). - ### Delete time spent -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356796) in GitLab 15.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356796) in GitLab 14.10. +> - Delete button [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356796) in GitLab 15.1. A timelog is a single entry of time spent, either positive or negative. @@ -124,7 +123,18 @@ Prerequisites: - You must be the author of the timelog or have at least the Maintainer role for the project. -You can [delete timelogs](../../api/graphql/reference/index.md#mutationtimelogdelete) using the GraphQL API. +To delete a timelog, either: + +- In the time tracking report, on the right of a timelog entry, select **Delete time spent** (**{remove}**). +- Use the [GraphQL API](../../api/graphql/reference/index.md#mutationtimelogdelete). + +### Delete all the time spent + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To delete all the time spent at once, use the `/remove_time_spent` [quick action](quick_actions.md). ## View a time tracking report @@ -137,7 +147,7 @@ To view a time tracking report: 1. Go to an issue or a merge request. 1. In the right sidebar, select **Time tracking report**. - + The breakdown of spent time is limited to a maximum of 100 entries. diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 9db30ee2ab6..facaba45aec 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -19,7 +19,7 @@ and from merge requests: - *When viewing a file, or the repository file list* - 1. In the upper right corner of the page, select **Open in Web IDE** if it is visible. 1. If **Open in Web IDE** is not visible: - 1. Select the **(angle-down)** next to **Edit** or **Gitpod**, depending on your configuration. + 1. Select the (**{chevron-lg-down}**) next to **Edit** or **Gitpod**, depending on your configuration. 1. Select **Open in Web IDE** from the list to display it as the editing option. 1. Select **Open in Web IDE** to open the editor. - *When viewing a merge request* - @@ -198,13 +198,13 @@ left. ## Switching merge requests -To switch between your authored and assigned merge requests, click the +To switch between your authored and assigned merge requests, select the dropdown in the top of the sidebar to open a list of merge requests. You must commit or discard all your changes before switching to a different merge request. ## Switching branches -To switch between branches of the current project repository, click the dropdown +To switch between branches of the current project repository, select the dropdown in the top of the sidebar to open a list of branches. You must commit or discard all your changes before switching to a different branch. @@ -320,7 +320,7 @@ The [File Sync](#file-syncing-to-web-terminal) feature is supported on Kubernete To enable the Web IDE terminals you must create the file `.gitlab/.gitlab-webide.yml` inside the repository's root. This -file is fairly similar to the [CI configuration file](../../../ci/yaml/index.md) +file is fairly similar to the [`.gitlab-ci.yml` file](../../../ci/yaml/index.md) syntax but with some restrictions: - No global blocks (such as `before_script` or `after_script`) can be defined. @@ -334,7 +334,7 @@ syntax but with some restrictions: that, if you override the default `script` value, you have to add a command which would keep the job running, like `sleep`. -In the code below there is an example of this configuration file: +For example, with this configuration file: ```yaml terminal: @@ -348,20 +348,20 @@ terminal: NODE_ENV: "test" ``` -After the terminal has started, the console is displayed and we could access +After the terminal starts, the console is displayed and you can access the project repository files. -When you use the image keyword, a container with the specified image is created. If you specify an image, it has no effect. This is the case when you use the [shell executor](https://docs.gitlab.com/runner/executors/shell.html). +When you use the `image` keyword, a container with the specified image is created. +If you use the [shell executor](https://docs.gitlab.com/runner/executors/shell.html) +or the [SSH executor](https://docs.gitlab.com/runner/executors/ssh.html), `image` has no effect. -**Important**. The terminal job is branch dependent. This means that the -configuration file used to trigger and configure the terminal is the one in -the selected branch of the Web IDE. - -If there is no configuration file in a branch, an error message is shown. +The terminal job is branch dependent. The configuration file used to trigger +and configure the terminal is the one in the selected branch of the Web IDE. +If no configuration file exists in a branch, an error message is displayed. ### Running interactive terminals in the Web IDE -If Interactive Terminals are available for the current user, the **Terminal** button is visible in the right sidebar of the Web IDE. Click this button to open +If Interactive Terminals are available for the current user, the **Terminal** button is visible in the right sidebar of the Web IDE. Select this button to open or close the terminal tab. After opening, the tab shows the **Start Web Terminal** button. This button may @@ -383,7 +383,7 @@ to running commands in a local terminal or through SSH. While the terminal is running, it can be stopped by clicking **Stop Terminal**. This disconnects the terminal and stops the runner's terminal job. From here, -click **Restart Terminal** to start a new terminal session. +select **Restart Terminal** to start a new terminal session. ### File syncing to web terminal diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index fe6c7ae62b8..5ae0cf46d9b 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -270,7 +270,7 @@ Support for displaying a generated table of contents with a custom side navigati Wikis are enabled by default in GitLab. Project [administrators](../../permissions.md) can enable or disable a project wiki by following the instructions in -[Sharing and permissions](../settings/index.md#sharing-and-permissions). +[Sharing and permissions](../settings/index.md#configure-project-visibility-features-and-permissions). Administrators for self-managed GitLab installs can [configure additional wiki settings](../../../administration/wikis/index.md). diff --git a/doc/user/project/working_with_projects.md b/doc/user/project/working_with_projects.md index bfc83aa22f5..83cab819f54 100644 --- a/doc/user/project/working_with_projects.md +++ b/doc/user/project/working_with_projects.md @@ -59,7 +59,7 @@ To explore project topics: The **Projects** page shows list of topics sorted by the number of associated projects. To view projects associated with a topic, select a topic from the list. -You can assign topics to a project on the [Project Settings page](settings/index.md#topics). +You can assign topics to a project on the [Project Settings page](settings/index.md#assign-topics-to-a-project). If you're an instance administrator, you can administer all project topics from the [Admin Area's Topics page](../admin_area/index.md#administering-topics). @@ -73,7 +73,7 @@ To create a project in GitLab: - Create a [blank project](#create-a-blank-project). - Create a project from a: - [built-in template](#create-a-project-from-a-built-in-template). - - [custom template](#create-a-project-from-a-custom-template). + - [custom template](#create-a-project-from-a-custom-template). - [HIPAA audit protocol template](#create-a-project-from-the-hipaa-audit-protocol-template). - [Import a project](../../user/project/import/index.md) from a different repository. Contact your GitLab administrator if this option is not available. @@ -115,7 +115,7 @@ Built-in templates are sourced from the following groups: - [`project-templates`](https://gitlab.com/gitlab-org/project-templates) - [`pages`](https://gitlab.com/pages) -Anyone can contribute a built-in template by following [these steps](https://about.gitlab.com/community/contribute/project-templates/). +Anyone can [contribute a built-in template](../../development/project_templates.md). To create a project from a built-in template: @@ -336,6 +336,29 @@ To view the activity of a project: 1. On the left sidebar, select **Project information > Activity**. 1. Select a tab to view the type of project activity. +## Search in projects + +You can search through your projects. + +1. On the top bar, select **Menu**. +1. In **Search your projects**, type the project name. + +GitLab filters as you type. + +You can also look for the projects you [starred](#star-a-project) (**Starred projects**). + +You can **Explore** all public and internal projects available in GitLab.com, from which you can filter by visibility, +through **Trending**, best rated with **Most stars**, or **All** of them. + +You can sort projects by: + +- Name +- Created date +- Updated date +- Owner + +You can also choose to hide or show archived projects. + ## Leave a project If you leave a project, you are no longer a project @@ -484,5 +507,5 @@ download starts, the `insteadOf` configuration sends the traffic to the secondar - [Import a project](../../user/project/import/index.md). - [Connect an external repository to GitLab CI/CD](../../ci/ci_cd_for_external_repos/index.md). - [Fork a project](repository/forking_workflow.md#creating-a-fork). -- [Adjust project visibility and access levels](settings/index.md#sharing-and-permissions). +- [Adjust project visibility and access levels](settings/index.md#configure-project-visibility-features-and-permissions). - [Limitations on project and group names](../../user/reserved_names.md#limitations-on-project-and-group-names) diff --git a/doc/user/search/advanced_search.md b/doc/user/search/advanced_search.md index 5435a9d027c..075c9b6154b 100644 --- a/doc/user/search/advanced_search.md +++ b/doc/user/search/advanced_search.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Global Search info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments" type: reference @@ -9,27 +9,23 @@ type: reference > Moved to GitLab Premium in 13.9. -NOTE: -This is the user documentation. To configure the Advanced Search, -visit the [administrator documentation](../../integration/elasticsearch.md). -Advanced Search is enabled in GitLab.com. +Advanced Search uses Elasticsearch for faster, more advanced search across the entire +GitLab instance. -GitLab Advanced Search expands on the Basic Search with an additional set of -features for faster, more advanced searches across the entire GitLab instance -when searching in: +Use Advanced Search when searching in: - Projects - Issues - Merge requests - Milestones -- Epics -- Comments +- Users +- Epics (when searching in a group only) - Code +- Comments - Commits -- Users - Wiki (except [group wikis](../project/wiki/group.md)) -The Advanced Search can be useful in various scenarios: +Advanced Search can be useful in various scenarios: - **Faster searches:** Advanced Search is based on Elasticsearch, which is a purpose-built full @@ -46,6 +42,13 @@ The Advanced Search can be useful in various scenarios: may be connected to each other, so your developers need to instantly search throughout the GitLab instance and find the code they search for. +## Configuring Advanced Search + +For self-managed GitLab instances, an administrator must +[configure Advanced Search](../../integration/advanced_search/elasticsearch.md). + +On GitLab.com, Advanced Search is enabled. + ## Advanced Search syntax See the documentation on [Advanced Search syntax](global_search/advanced_search_syntax.md). @@ -93,7 +96,7 @@ doesn't return any results for searches considered abusive according to the foll - Searches with less than 2 characters. - Searches with any term greater than 100 characters. URL search terms have a maximum of 200 characters. -- Searches with a stop word as the only term (ie: "the", "and", "if", etc.). +- Searches with a stop word as the only term (for example, "the", "and", "if", etc.). - Searches with a `group_id` or `project_id` parameter that is not completely numeric. - Searches with a `repository_ref` or `project_ref` parameter that has special characters not allowed by [Git refname](https://git-scm.com/docs/git-check-ref-format). - Searches with a `scope` that is unknown. diff --git a/doc/user/search/global_search/advanced_search_syntax.md b/doc/user/search/global_search/advanced_search_syntax.md index 3aa016f0bff..9daa7afd6de 100644 --- a/doc/user/search/global_search/advanced_search_syntax.md +++ b/doc/user/search/global_search/advanced_search_syntax.md @@ -1,5 +1,5 @@ --- -stage: Enablement +stage: Data Stores group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -18,7 +18,7 @@ Advanced Search searches default project branches only. ## General search -<!-- markdownlint-disable --> +<!-- markdownlint-disable --> | Use | Description | Example | |-----|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------| @@ -33,7 +33,7 @@ Advanced Search searches default project branches only. | Use | Description | Example | |--------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| -| `filename:` | File name | [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | +| `filename:` | Filename | [`filename:*spec.rb`](https://gitlab.com/search?snippets=&scope=blobs&repository_ref=&search=filename%3A*spec.rb&group_id=9970&project_id=278964) | | `path:` | Repository location | [`path:spec/workers/`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=path%3Aspec%2Fworkers&snippets=) | | `extension:` | File extension, without the `.` | [`extension:js`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=extension%3Ajs&snippets=) | | `blob:` | Git object ID | [`blob:998707*`](https://gitlab.com/search?snippets=false&scope=blobs&repository_ref=&search=blob%3A998707*&group_id=9970) | diff --git a/doc/user/search/img/basic_search_results.png b/doc/user/search/img/basic_search_results.png Binary files differdeleted file mode 100644 index 52aa2e3fe6c..00000000000 --- a/doc/user/search/img/basic_search_results.png +++ /dev/null diff --git a/doc/user/search/img/basic_search_results_v15_1.png b/doc/user/search/img/basic_search_results_v15_1.png Binary files differnew file mode 100644 index 00000000000..b85627c9b95 --- /dev/null +++ b/doc/user/search/img/basic_search_results_v15_1.png diff --git a/doc/user/search/img/basic_search_v14_4.png b/doc/user/search/img/basic_search_v14_4.png Binary files differdeleted file mode 100644 index 86aab68f1f0..00000000000 --- a/doc/user/search/img/basic_search_v14_4.png +++ /dev/null diff --git a/doc/user/search/img/basic_search_v15_1.png b/doc/user/search/img/basic_search_v15_1.png Binary files differnew file mode 100644 index 00000000000..069d62ca80c --- /dev/null +++ b/doc/user/search/img/basic_search_v15_1.png diff --git a/doc/user/search/img/code_search_git_blame_v14_9.png b/doc/user/search/img/code_search_git_blame_v14_9.png Binary files differdeleted file mode 100644 index eb8d14de4a4..00000000000 --- a/doc/user/search/img/code_search_git_blame_v14_9.png +++ /dev/null diff --git a/doc/user/search/img/code_search_git_blame_v15_1.png b/doc/user/search/img/code_search_git_blame_v15_1.png Binary files differnew file mode 100644 index 00000000000..e61ee5993c2 --- /dev/null +++ b/doc/user/search/img/code_search_git_blame_v15_1.png diff --git a/doc/user/search/img/dashboard_links_v14_6.png b/doc/user/search/img/dashboard_links_v14_6.png Binary files differdeleted file mode 100644 index 52ae39d9d1a..00000000000 --- a/doc/user/search/img/dashboard_links_v14_6.png +++ /dev/null diff --git a/doc/user/search/img/issues_mrs_shortcut_v14_6.png b/doc/user/search/img/issues_mrs_shortcut_v14_6.png Binary files differdeleted file mode 100644 index 52753eb8fc7..00000000000 --- a/doc/user/search/img/issues_mrs_shortcut_v14_6.png +++ /dev/null diff --git a/doc/user/search/img/multiple_assignees.png b/doc/user/search/img/multiple_assignees.png Binary files differdeleted file mode 100644 index 5c46f3dda46..00000000000 --- a/doc/user/search/img/multiple_assignees.png +++ /dev/null diff --git a/doc/user/search/img/search_issues_board.png b/doc/user/search/img/search_issues_board.png Binary files differdeleted file mode 100644 index 84048ae6a02..00000000000 --- a/doc/user/search/img/search_issues_board.png +++ /dev/null diff --git a/doc/user/search/index.md b/doc/user/search/index.md index 171d8a63d2d..76a85b55585 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -1,80 +1,82 @@ --- -stage: Create -group: Editor +stage: Data Stores +group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Searching in GitLab **(FREE)** -## Search issues and merge requests +GitLab has two types of searches available: _basic_ and _advanced_. -To search through issues and merge requests in multiple projects, on the top bar, select the **Issues** or **Merge requests** links. +Both types of search are the same, except when you are searching through code. -The numbers indicate how many issues, merge requests, and to-do items are assigned to you: +- When you use basic search to search code, your search includes one project at a time. +- When you use [advanced search](advanced_search.md) to search code, your search includes all projects at once. - +## Basic search + +Use basic search to find: + +- Projects +- Issues +- Merge requests +- Milestones +- Users +- Epics (when searching in a group only) +- Code +- Comments +- Commits +- Wiki + +## Perform a search + +To start a search, type your search query in the search bar on the top-right of the screen. +You must type at least two characters. + + + +After the results are displayed, you can modify the search, select a different type of data to +search, or choose a specific group or project. + + + +## Search in code + +To search through code or other documents in a project: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the top bar, in the search field, type the string you want to search for. +1. Press **Enter**. + +Code search shows only the first result in the file. -- **{issues}** **Issues**: Issues assigned to you. -- **{merge-request-open}** **Merge requests**: Open [merge requests](../project/merge_requests/index.md). - Select the icon to show a dropdown list of merge request filters: - - [Attention requests](../project/merge_requests/index.md#request-attention-to-a-merge-request) (**{attention-solid}**) for you. - - [Review requests](../project/merge_requests/reviews/index.md) for you. - - Merge requests assigned to you. -- **{todo-done}** **To-do items**: The [to-do items](../todos.md) assigned to you. +To search across all of GitLab, ask your administrator to enable [advanced search](advanced_search.md). -You can search through **Open**, **Closed**, or **All** issues. +### View Git blame from code search -You can also filter the results using the search and filter field, as described in -[Filter issue and merge request lists](#filter-issue-and-merge-request-lists). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327052) in GitLab 14.7. -### Issues and MRs assigned to you or created by you +After you find search results, you can view who made the last change to the line +where the results were found. -GitLab shows shortcuts to issues and merge requests created by you or assigned to you -in the search field in the upper right corner: +1. From the code search result, hover over the line number. +1. On the left, select **View blame**. - +  -### Filter issue and merge request lists +## Search for a SHA -> - Filtering by epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195704) in GitLab 12.9. -> - Filtering by child epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9029) in GitLab 13.0. -> - Filtering by iterations was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118742) in GitLab 13.6. -> - Filtering by iterations was moved from GitLab Ultimate to GitLab Premium in 13.9. -> - Filtering by type was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322755) in GitLab 13.10 [with a flag](../../administration/feature_flags.md) named `vue_issues_list`. Disabled by default. -> - Filtering by type was [enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/322755) in GitLab 14.10. -> - Filtering by attention request was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343528) in GitLab 14.10 [with a flag](../../administration/feature_flags.md) named `mr_attention_requests`. Disabled by default. +You can search for a commit SHA. -Follow these steps to filter the **Issues** and **Merge requests** list pages in projects and -groups: +1. On the top bar, select **Menu > Projects** and find your project. +1. On the top bar, in the search field, type the SHA. -1. Select **Search or filter results...**. -1. In the dropdown list that appears, select the attribute you wish to filter by: - - Assignee - - [Attention requests](../project/merge_requests/index.md#request-attention-to-a-merge-request) - - Author - - Confidential - - [Epic and child Epic](../group/epics/index.md) (available only for the group the Epic was created, not for [higher group levels](https://gitlab.com/gitlab-org/gitlab/-/issues/233729)). - - [Iteration](../group/iterations/index.md) - - [Label](../project/labels.md) - - [Milestone](../project/milestones/index.md) - - My-reaction - - Release - - Type - - Weight - - Search for this text -1. Select or type the operator to use for filtering the attribute. The following operators are - available: - - `=`: Is - - `!=`: Is not ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18059) in GitLab 12.7) -1. Enter the text to [filter the attribute by](#filters-autocomplete). - You can filter some attributes by **None** or **Any**. -1. Repeat this process to filter by multiple attributes. Multiple attributes are joined by a logical - `AND`. +If a single result is returned, GitLab redirects to the commit result +and gives you the option to return to the search results page. -GitLab displays the results on-screen, but you can also -[retrieve them as an RSS feed](#retrieve-search-results-as-feed). + -### Searching for specific terms +## Searching for specific terms You can filter issues and merge requests by specific terms included in titles or descriptions. @@ -87,7 +89,7 @@ You can filter issues and merge requests by specific terms included in titles or issues for `included in titles` is same as `included titles` - Search is limited to 4096 characters and 64 terms per query. -### Retrieve search results as feed +## Retrieve search results as feed > Feeds for merge requests were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66336) in GitLab 14.3. @@ -96,92 +98,19 @@ RSS feed of search results: 1. Go to your project's page. 1. On the left sidebar, select **Issues** or **Merge requests**. -1. Build your search query as described in [Filter issue and merge request lists](#filter-issue-and-merge-request-lists). +1. Perform a search. 1. Select the feed symbol **{rss}** to display the results as an RSS feed in Atom format. The URL of the result contains both a feed token, and your search query. You can add this URL to your feed reader. -### Filtering by ID - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39908) in GitLab 12.1. - -You can filter the **Issues** list to individual instances by their ID. For example, enter filter `#10` to return only issue 10. The same applies to the **Merge requests** list. Enter filter `#30` to return only merge request 30. - - - -### Filtering merge requests by approvers **(PREMIUM)** - -> Moved to GitLab Premium in 13.9. - -To filter merge requests by an individual eligible approver ([Codeowner](../project/code_owners.md)), you can type (or select from -the dropdown list) **Approver** and select the user. - - - -### Filtering merge requests by "approved by" **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30335) in GitLab 13.0. -> - Moved to GitLab Premium in 13.9. - -To filter merge requests already approved by a specific individual, you can type (or select from -the dropdown list) **Approved-By** and select the user. - - - -### Filtering merge requests by reviewer - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47605) in GitLab 13.7. - -To filter review requested merge requests for a specific individual, you can type (or select from -the dropdown list) **Reviewer** and select the user. - -### Filtering merge requests by environment or deployment date - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44041) in GitLab 13.6. - -To filter merge requests by deployment data, such as the environment or a date, -you can type (or select from the dropdown list) the following: - -- Environment -- Deployed-before -- Deployed-after - -NOTE: -Projects using a [fast-forward merge method](../project/merge_requests/fast_forward_merge.md) -do not return results, as this method does not create a merge commit. - -When filtering by an environment, a dropdown list presents all environments that -you can choose from: - - - -When filtering by `Deployed-before` or `Deployed-after`, the date refers to when -the deployment to an environment (triggered by the merge commit) completed successfully. -You must enter the deploy date manually. Deploy dates -use the format `YYYY-MM-DD`, and must be quoted if you wish to specify -both a date and time (`"YYYY-MM-DD HH:MM"`): - - - -## Filters autocomplete - -GitLab provides many filters across many pages (issues, merge requests, epics, -and pipelines among others) which you can use to narrow down your search. When -using the filter functionality, you can start typing characters to bring up -relevant users or other attributes. - -For performance optimization, there is a requirement of a minimum of three -characters to begin your search. To search for issues with the assignee `Simone Presley`, -you must type at least `Sim` before autocomplete displays results. - ## Search history Search history is available for issues and merge requests, and is stored locally in your browser. To run a search from history: 1. In the top menu, select **Issues** or **Merge requests**. -1. To the left of the search bar, click **Recent searches**, and select a search from the list. +1. To the left of the search bar, select **Recent searches**, and select a search from the list. ## Removing search filters @@ -189,61 +118,6 @@ Individual filters can be removed by clicking on the filter's (x) button or back To delete filter tokens one at a time, the <kbd>⌥</kbd> (Mac) / <kbd>Control</kbd> + <kbd>⌫</kbd> keyboard combination can be used. -## Filtering with multiple filters of the same type - -Some filters can be added multiple times. These include but are not limited to assignees and labels. When you filter with these multiple filters of the same type, the `AND` logic is applied. For example, if you were filtering `assignee:@sam assignee:@sarah`, your results include only entries whereby the assignees are assigned to both Sam and Sarah are returned. - - - -## To-Do List - -You can search your [To-Do List](../todos.md) by "to do" and "done". -You can filter to-do items per project, author, type, and action. -Also, you can sort them by [**Label priority**](../../user/project/labels.md#set-label-priority), -**Last created**, and **Oldest created**. - -## Projects - -You can search through your projects from the top bar, by selecting **Menu > Projects**. -On the field **Filter by name**, type the project or group name you want to find, and GitLab -filters them for you as you type. - -You can also look for the projects you [starred](../project/working_with_projects.md#star-a-project) (**Starred projects**). -You can **Explore** all public and internal projects available in GitLab.com, from which you can filter by visibility, -through **Trending**, best rated with **Most stars**, or **All** of them. - -You can also sort them by: - -- Name -- Created date -- Updated date -- Owner - -You can also choose to hide or show archived projects. - -## Groups - -Similarly to [projects search](#projects), you can search through your groups from -the left menu, by clicking the menu bar, then **Groups**. - -On the field **Filter by name**, type the group name you want to find, and GitLab -filters them for you as you type. - -You can also **Explore** all public and internal groups available in GitLab.com, -and sort them by **Name**, **Last created**, **Oldest created**, or **Updated date**. - -## Issue boards - -From an [issue board](../../user/project/issue_board.md), you can filter issues by **Author**, **Assignee**, **Milestone**, and **Labels**. -You can also filter them by name (issue title), from the field **Filter by name**, which is loaded as you type. - -To search for issues to add to lists present in your issue board, select -the button **Add issues** on the top-right of your screen, opening a modal window from which -you can, besides filtering them by **Name**, **Author**, **Assignee**, **Milestone**, -and **Labels**, select multiple issues to add to a list of your choice: - - - ## Autocomplete suggestions In the search bar, you can view autocomplete suggestions for: @@ -257,63 +131,6 @@ In the search bar, you can view autocomplete suggestions for: - Recently viewed epics (try and type some word from the title of a recently viewed epic) - [GitLab Flavored Markdown](../markdown.md#gitlab-specific-references) (GLFM) for issues in a project (try and type a GLFM reference for an issue) -## Basic search - -The Basic search in GitLab enables you to search -across the entire GitLab instance, in a group, or in a single project. Basic search is -backed by the database and allows searching in: - -- Projects -- Issues -- Merge requests -- Milestones -- Users -- Epics (Group only) -- Code (Project only) -- Comments (Project only) -- Commits (Project only) -- Wiki (Project only) - -To start a search, type into the search bar on the top-right of the screen. You can always search -in all GitLab and may also see the options to search in a group or project if you are in the -group or project dashboard. - - - -After the results are returned, you can modify the search, select a different type of data to -search, or choose a specific group or project. - - - -### Code search - -To search through code or other documents in a single project, you can use -the search field on the top-right of your screen while the project page is open. -Code search shows only the first result in the file. - -#### Git blame from code search **(FREE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327052) in GitLab 14.7. - -You can access Git blame from any line that returned a result from the code search: - - - -### SHA search - -You can quickly access a commit from the project dashboard by entering the SHA -into the search field on the top right of the screen. If a single result is found, you are -redirected to the commit result and given the option to return to the search results page. - - - -## Advanced Search **(PREMIUM)** - -Leverage Elasticsearch for faster, more advanced code search across your entire -GitLab instance. - -[Learn how to use the Advanced Search.](advanced_search.md) - ## Search settings > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292941) in GitLab 13.8 [with a flag](../../administration/feature_flags.md) named `search_settings_in_page`. Disabled by default. diff --git a/doc/user/tasks.md b/doc/user/tasks.md index 8be7fbca213..fc49661c61c 100644 --- a/doc/user/tasks.md +++ b/doc/user/tasks.md @@ -8,10 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334812) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `work_items`. Disabled by default. -WARNING: -Tasks are in [**Alpha**](../policy/alpha-beta-support.md#alpha-features). Current implementation does not have any API requests and works with mocked data. -For the latest updates, check the [Tasks Roadmap](https://gitlab.com/groups/gitlab-org/-/epics/7103). - FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `work_items`. @@ -31,7 +27,26 @@ to work items and adding custom work item types, visit [epic 6033](https://gitlab.com/groups/gitlab-org/-/epics/6033) or [Plan direction page](https://about.gitlab.com/direction/plan/). -## View a task +## Create a task + +To create a task: + +1. In an issue description, create a [task list](markdown.md#task-lists). +1. Hover over a task item and select **Convert to work item** (**{doc-new}**). +1. Confirm or edit the title, and select **Create work item**. + +## Edit a task + +To edit a task: + +1. In the issue description, view the task links. +1. Select a link. The task is displayed. + - To edit the description, select **Edit**, then select **Save**. + - To edit the title or state, make your changes, then click outside the field. The changes are saved automatically. + +## Delete a task + +To delete a task: -The only way to view a task is to open it with a deep link, -for example: `/<group_name>/<project_name>/-/work_item/1`. +1. In the issue description, select the task. +1. From the options menu (**{ellipsis_v}**), select **Delete work item**. diff --git a/doc/user/todos.md b/doc/user/todos.md index c261d719da0..0a120100c57 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -23,6 +23,14 @@ To access your To-Do List: On the top bar, in the top right, select To-Do List (**{task-done}**). +## Search the To-Do List + +You can search your To-Do List by `to do` and `done`. + +You can filter to-do items per project, author, type, and action. +Also, you can sort them by [**Label priority**](project/labels.md#set-label-priority), +**Last created**, and **Oldest created**. + ## Actions that create to-do items Many to-do items are created automatically. diff --git a/doc/user/upgrade_email_bypass.md b/doc/user/upgrade_email_bypass.md index 8cd9a47f3e6..6401828270d 100644 --- a/doc/user/upgrade_email_bypass.md +++ b/doc/user/upgrade_email_bypass.md @@ -27,7 +27,7 @@ sent within five minutes, with a link for users to re-confirm the subject email ## Do the confirmation emails expire? -The links in these re-confirmation emails expire after one day by default. Users who click an expired link are asked to request a new re-confirmation email. Any user can request a new re-confirmation email from `http://gitlab.example.com/users/confirmation/new`. +The links in these re-confirmation emails expire after one day by default. Users who select an expired link are asked to request a new re-confirmation email. Any user can request a new re-confirmation email from `http://gitlab.example.com/users/confirmation/new`. ## Generate a list of affected users diff --git a/doc/user/usage_quotas.md b/doc/user/usage_quotas.md index 21aa93d3f8b..84c98a60917 100644 --- a/doc/user/usage_quotas.md +++ b/doc/user/usage_quotas.md @@ -32,7 +32,7 @@ You can view storage usage for your project or [namespace](../user/group/#namesp The statistics are displayed. Select any title to view details. The information on this page is updated every 90 minutes. -If your namespace shows `N/A`, push a commit to any project in the +If your namespace shows `'Not applicable.'`, push a commit to any project in the namespace to recalculate the storage. ## Storage usage statistics @@ -50,7 +50,7 @@ The following storage usage statistics are available to a maintainer: ## Manage your storage usage -You can use several methods to manage and reduce your usage for some storage types. +You can use several methods to manage and reduce your usage for some storage types. For more information, see the following pages: |
