diff options
Diffstat (limited to 'doc')
724 files changed, 18315 insertions, 15181 deletions
diff --git a/doc/.vale/gitlab/Admin.yml b/doc/.vale/gitlab/Admin.yml index f6b0a988499..78a86e27456 100644 --- a/doc/.vale/gitlab/Admin.yml +++ b/doc/.vale/gitlab/Admin.yml @@ -1,5 +1,5 @@ --- -# Warning: gitlab.Admin +# Suggestion: gitlab.Admin # # Checks for "admin" and recommends using the full word instead. "Admin Area" is OK. # diff --git a/doc/.vale/gitlab/BadPlurals.yml b/doc/.vale/gitlab/BadPlurals.yml index 63f002fec94..533805c67b0 100644 --- a/doc/.vale/gitlab/BadPlurals.yml +++ b/doc/.vale/gitlab/BadPlurals.yml @@ -1,5 +1,5 @@ --- -# Suggestion: gitlab.BadPlurals +# Warning: gitlab.BadPlurals # # Don't write plural words with the '(s)' construction. "HTTP(S)" is acceptable. # diff --git a/doc/.vale/gitlab/EOLWhitespace.yml b/doc/.vale/gitlab/EOLWhitespace.yml index 483db0cafe6..e160b706014 100644 --- a/doc/.vale/gitlab/EOLWhitespace.yml +++ b/doc/.vale/gitlab/EOLWhitespace.yml @@ -1,5 +1,5 @@ --- -# Error: gitlab.EOLWhitespace +# Warning: gitlab.EOLWhitespace # # Checks that there is no useless whitespace at the end of lines. # diff --git a/doc/.vale/gitlab/FutureTense.yml b/doc/.vale/gitlab/FutureTense.yml index fc414a9c0fd..64e79612fff 100644 --- a/doc/.vale/gitlab/FutureTense.yml +++ b/doc/.vale/gitlab/FutureTense.yml @@ -1,5 +1,5 @@ --- -# Suggestion: gitlab.FutureTense +# Warning: gitlab.FutureTense # # Checks for use of future tense in sentences. Present tense is strongly preferred. # diff --git a/doc/.vale/gitlab/HeadingContent.yml b/doc/.vale/gitlab/HeadingContent.yml index a8dc596f2a2..c2dd2a5c6c2 100644 --- a/doc/.vale/gitlab/HeadingContent.yml +++ b/doc/.vale/gitlab/HeadingContent.yml @@ -1,5 +1,5 @@ --- -# Error: gitlab.HeadingContent +# Warning: gitlab.HeadingContent # # Checks for generic, unhelpful subheadings. # diff --git a/doc/.vale/gitlab/HeadingDepth.yml b/doc/.vale/gitlab/HeadingDepth.yml new file mode 100644 index 00000000000..466ab317226 --- /dev/null +++ b/doc/.vale/gitlab/HeadingDepth.yml @@ -0,0 +1,13 @@ +--- +# Warning: gitlab.HeadingDepth +# +# Checks that there are no headings greater than 3 levels +# +# For a list of all options, see https://vale.sh/docs/topics/styles/ +extends: existence +message: 'The subheading "%s" is nested too deeply. Headings deeper than H5 suggest the section or page should be refactored.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#headings-in-markdown +level: warning +scope: raw +raw: + - '(?<=\n)#{5,}\s.*' diff --git a/doc/.vale/gitlab/MultiLineLinks.yml b/doc/.vale/gitlab/MultiLineLinks.yml new file mode 100644 index 00000000000..64ad017f16c --- /dev/null +++ b/doc/.vale/gitlab/MultiLineLinks.yml @@ -0,0 +1,14 @@ +--- +# Error: gitlab.MultiLineLinks +# +# Checks that links are all on a single line. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Link "%s" must be on a single line, even if very long.' +link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#basic-link-criteria +level: error +scope: raw +raw: + - '\[[^\[\]]*?\n[^\[\]]*?\]\([^\)]*?\)|' + - '\[[^\[\]]*?\]\([^\)]*?\n[^\)]*\)' diff --git a/doc/.vale/gitlab/OutdatedVersions.yml b/doc/.vale/gitlab/OutdatedVersions.yml index c78c045a1fd..f25de44ad17 100644 --- a/doc/.vale/gitlab/OutdatedVersions.yml +++ b/doc/.vale/gitlab/OutdatedVersions.yml @@ -1,5 +1,5 @@ --- -# Warning: gitlab.OutdatedVersions +# Suggestion: gitlab.OutdatedVersions # # Checks for references to versions of GitLab that are no longer supported. # diff --git a/doc/.vale/gitlab/Possessive.yml b/doc/.vale/gitlab/Possessive.yml index 158c689cac1..92ae66543a2 100644 --- a/doc/.vale/gitlab/Possessive.yml +++ b/doc/.vale/gitlab/Possessive.yml @@ -1,5 +1,5 @@ --- -# Warning: gitlab.Possessive +# Error: gitlab.Possessive # # The word GitLab should not be used in the possessive form. # diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index a8519d898db..dc05aa05730 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -56,6 +56,7 @@ exceptions: - DML - DNS - DOM + - DORA - DSA - DSL - DVCS diff --git a/doc/.vale/gitlab/VersionText.yml b/doc/.vale/gitlab/VersionText.yml index 68753de60aa..571fba52ab7 100644 --- a/doc/.vale/gitlab/VersionText.yml +++ b/doc/.vale/gitlab/VersionText.yml @@ -1,23 +1,19 @@ --- # Error: gitlab.VersionText # -# Checks that version text is formatted correctly. +# Checks that multi-line version text is formatted correctly. # -# Specifically looks for either of the following that is immediately followed on the next line -# by content, which will break rendering: +# Specifically, looks for multi-line version text that doesn't use `-` to make it a list. +# For example: # -# - `> Introduced` (version text without a link) -# - `> [Introduced` (version text with a link) -# -# Because it excludes the prefix `> - `, it doesn't look for multi-line version text, for which -# content immediately on the next line is ok. However, this will often highlight where multi-line -# version text is attempted without `-` characters. +# - `> Introduced in GitLab 14.0. +# - `> Removed in GitLab 15.0. # # For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles extends: existence -message: 'This introduced-in line is not formatted correctly.' +message: 'This introduced-in section is not formatted correctly. Each entry must start with `> -` and long entries must be on one line.' link: https://docs.gitlab.com/ee/development/documentation/versions.html level: error scope: raw raw: - - '> \[?Introduced.+\n[^\n]' + - '\n#.*\n\n> [^-].+\n[^\n`]' diff --git a/doc/.vale/gitlab/VersionTextSingleLine.yml b/doc/.vale/gitlab/VersionTextSingleLine.yml new file mode 100644 index 00000000000..f76574bcf8a --- /dev/null +++ b/doc/.vale/gitlab/VersionTextSingleLine.yml @@ -0,0 +1,13 @@ +--- +# Error: gitlab.VersionTextSingleLine +# +# Verifies that single-item version notes don't have a hyphen. +# +# For a list of all options, see https://errata-ai.gitbook.io/vale/getting-started/styles +extends: existence +message: 'Version text with only a single item must not start with a hyphen.' +link: https://docs.gitlab.com/ee/development/documentation/versions.html#add-a-version-history-item +level: error +scope: raw +raw: + - '(\r|\n|\r\n){2}(> - .*)(\r|\n|\r\n){2}' diff --git a/doc/administration/application_settings_cache.md b/doc/administration/application_settings_cache.md index 6c58e6886c4..30fd9ab85a8 100644 --- a/doc/administration/application_settings_cache.md +++ b/doc/administration/application_settings_cache.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Memory +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 817f22debbc..3bb0ce41861 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -6,44 +6,53 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Audit event streaming **(ULTIMATE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events_namespace`. Disabled by default. -> - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338939) in GitLab 14.7. -> - [Feature flag `ff_external_audit_events_namespace`](https://gitlab.com/gitlab-org/gitlab/-/issues/349588) removed in GitLab 14.8. +> - API [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `ff_external_audit_events_namespace`. Disabled by default. +> - API [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/338939) in GitLab 14.7. +> - API [Feature flag `ff_external_audit_events_namespace`](https://gitlab.com/gitlab-org/gitlab/-/issues/349588) removed in GitLab 14.8. +> - UI [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336411) in GitLab 14.9. > - [Subgroup events recording](https://gitlab.com/gitlab-org/gitlab/-/issues/366878) fixed in GitLab 15.2. - -Users can set an HTTP endpoint for a top-level group to receive all audit events about the group, its subgroups, and +> - Custom HTTP headers API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361216) in GitLab 15.1 [with a flag](feature_flags.md) named `streaming_audit_event_headers`. Disabled by default. +> - Custom HTTP headers API [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) in GitLab 15.2. +> - Custom HTTP headers API [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/366524) in GitLab 15.3. [Feature flag `streaming_audit_event_headers`](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) removed. +> - Custom HTTP headers UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. +> - Custom HTTP headers UI [made generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) in GitLab 15.3. [Feature flag `custom_headers_streaming_audit_events_ui`](https://gitlab.com/gitlab-org/gitlab/-/issues/365259) removed. +> - [Improved user experience](https://gitlab.com/gitlab-org/gitlab/-/issues/367963) in GitLab 15.4. + +Users can set a streaming destination for a top-level group to receive all audit events about the group, its subgroups, and projects as structured JSON. Top-level group owners can manage their audit logs in third-party systems. Any service that can receive -structured JSON data can be used as the endpoint. +structured JSON data can be used as the streaming destination. + +Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. NOTE: GitLab can stream a single event more than once to the same destination. Use the `id` key in the payload to deduplicate incoming data. -## Add a new event streaming destination +## Add a new streaming destination WARNING: -Event streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the destination endpoint. +Streaming destinations receive **all** audit event data, which could include sensitive information. Make sure you trust the streaming destination. ### Use the GitLab UI -Users with at least the Owner role for a group can add event streaming destinations for it: +Users with the Owner role for a group can add streaming destinations for it: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Audit events**. 1. On the main area, select **Streams** tab. - - When the destination list is empty, select **Add stream** to show the section for adding destinations. - - When the destination list is not empty, select **{plus}** to show the section for adding destinations. -1. Enter the destination URL to add and select **Add**. - -Event streaming is enabled if: - -- No warning is shown. -- The added endpoint is displayed in the UI. +1. Select **Add streaming destination** to show the section for adding destinations. +1. Enter the destination URL to add. +1. Optional. Locate the **Custom HTTP headers** table. +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the **Active** checkbox, see the + [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. After all headers have been filled out, select **Add** to add the new streaming destination. ### Use the API -To enable event streaming and add a destination, users with at least the Owner role for a group must use the +To enable streaming and add a destination, users with the Owner role for a group must use the `externalAuditEventDestinationCreate` mutation in the GraphQL API. ```graphql @@ -51,6 +60,7 @@ mutation { externalAuditEventDestinationCreate(input: { destinationUrl: "https://mydomain.io/endpoint/ingest", groupPath: "my-group" } ) { errors externalAuditEventDestination { + id destinationUrl verificationToken group { @@ -66,23 +76,35 @@ Event streaming is enabled if: - The returned `errors` object is empty. - The API responds with `200 OK`. +Group owners can add an HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. You can retrieve the destination ID +by [listing all the streaming destinations](#use-the-api-1) for the group or from the mutation above. + +```graphql +mutation { + auditEventsStreamingHeadersCreate(input: { destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601", key: "foo", value: "bar" }) { + errors + } +} +``` + +The header is created if the returned `errors` object is empty. + ## List streaming destinations -Users with at least the Owner role for a group can list event streaming destinations. +Users with the Owner role for a group can list streaming destinations. ### Use the GitLab UI -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336411) in GitLab 14.9. - -Users with at least the Owner role for a group can list event streaming destinations: +To list the streaming destinations: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Audit events**. 1. On the main area, select **Streams** tab. +1. To the right of the item, select **Edit** (**{pencil}**) to see all the custom HTTP headers. ### Use the API -Users with at least the Owner role for a group can view a list of event streaming destinations at any time using the +Users with the Owner role for a group can view a list of streaming destinations at any time using the `externalAuditEventDestinations` query type. ```graphql @@ -94,42 +116,48 @@ query { destinationUrl verificationToken id + headers { + nodes { + key + value + id + } + } } } } } ``` -If the resulting list is empty, then audit event streaming is not enabled for that group. +If the resulting list is empty, then audit streaming is not enabled for that group. -## Delete streaming destinations +You need the ID values returned by this query for the update and delete mutations. -Users with at least the Owner role for a group can delete event streaming destinations using the -`deleteAuditEventDestinations` mutation type. +## Update streaming destinations -When the last destination is successfully deleted, event streaming is disabled for the group. +Users with the Owner role for a group can update streaming destinations. ### Use the GitLab UI -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336411) in GitLab 14.9. - -Users with at least the Owner role for a group can delete event streaming destinations. +To update a streaming destinations custom HTTP headers: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Audit events**. 1. On the main area, select **Streams** tab. -1. Select **{remove}** at the right side of each item. - -The external streaming destination is deleted when: - -- No warning is shown. -- The deleted endpoint is not displayed in the UI. +1. To the right of the item, select **Edit** (**{pencil}**). +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to update. +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the **Active** checkbox, see the + [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). +1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to + 20 headers per streaming destination. +1. Select **Save** to update the streaming destination. ### Use the API -Delete an event streaming destination by specifying an ID. Get the required ID by -[listing the details](audit_event_streaming.md#use-the-api-1) of event -streaming destinations. +Users with the Owner role for a group can update streaming destinations custom HTTP headers using the +`auditEventsStreamingHeadersUpdate` mutation type. You can retrieve the custom HTTP headers ID +by [listing all the custom HTTP headers](#use-the-api-1) for the group. ```graphql mutation { @@ -139,89 +167,71 @@ mutation { } ``` -Destination is deleted if: +Streaming destination is updated if: - The returned `errors` object is empty. - The API responds with `200 OK`. -## Custom HTTP headers - -> - API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361216) in GitLab 15.1 [with a flag](feature_flags.md) named `streaming_audit_event_headers`. Disabled by default. -> - API [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/362941) in GitLab 15.2. -> - UI [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361630) in GitLab 15.2 [with a flag](feature_flags.md) named `custom_headers_streaming_audit_events_ui`. Disabled by default. - -FLAG: -On self-managed GitLab, by default the API for this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `streaming_audit_event_headers`. -On GitLab.com, the API for this feature is available. - -Each streaming destination can have up to 20 custom HTTP headers included with each streamed event. - -### Adding custom HTTP headers - -Add custom HTTP headers with the API or GitLab UI. - -#### Use the API - -Group owners can add a HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. You can retrieve the destination ID -by [listing the external audit destinations](#list-streaming-destinations) on the group. +Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID +by [listing all the custom HTTP headers](#use-the-api-1) for the group. ```graphql mutation { - auditEventsStreamingHeadersCreate(input: { destinationId: "gid://gitlab/AuditEvents::ExternalAuditEventDestination/24601", key: "foo", value: "bar" }) { + auditEventsStreamingHeadersDestroy(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/1" }) { errors } } ``` -The header is created if the returned `errors` object is empty. +The header is deleted if the returned `errors` object is empty. -#### Use the GitLab UI +## Delete streaming destinations -FLAG: -On self-managed GitLab, by default the UI for this feature is not available. To make it available per group, ask an administrator to -[enable the feature flag](../administration/feature_flags.md) named `custom_headers_streaming_audit_events_ui`. On GitLab.com, the UI for this feature is -not available. The UI for this feature is not ready for production use. +Users with the Owner role for a group can delete streaming destinations. -Users with at least the Owner role for a group can add event streaming destinations and custom HTTP headers for it: +When the last destination is successfully deleted, streaming is disabled for the group. + +### Use the GitLab UI + +To delete a streaming destination: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Audit events**. -1. On the main area, select **Streams** tab. - - When the destination list is empty, select **Add stream** to show the section for adding destinations. - - When the destination list is not empty, select **{plus}** to show the section for adding destinations. -1. Enter the destination URL to add. -1. Locate the **Custom HTTP headers** table. -1. In the **Header** column, add the header's name. -1. In the **Value** column, add the header's value. -1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the **Active** checkbox, see the - [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). -1. Enter as many name and value pairs as required. When you enter a unique name and a value for a header, a new row in the table automatically appears. You can add up to - 20 headers per endpoint. -1. After all headers have been filled out, select **Add** to add the new endpoint. - -Event streaming is enabled if: +1. On the main area, select the **Streams** tab. +1. To the right of the item, select **Delete** (**{remove}**). -- No warning is shown. -- The added endpoint is displayed in the UI. +To delete only the custom HTTP headers for a streaming destination: -### Updating custom HTTP headers +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the main area, select the **Streams** tab. +1. To the right of the item, **Edit** (**{pencil}**). +1. Locate the **Custom HTTP headers** table. +1. Locate the header that you wish to remove. +1. To the right of the header, select **Delete** (**{remove}**). +1. Select **Save** to update the streaming destination. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/361964) in GitLab 15.2. +### Use the API -Group owners can update a HTTP header using the GraphQL `auditEventsStreamingHeadersCreate` mutation. +Users with the Owner role for a group can delete streaming destinations using the +`externalAuditEventDestinationDestroy` mutation type. You can retrieve the destinations ID +by [listing all the streaming destinations](#use-the-api-1) for the group. ```graphql mutation { - auditEventsStreamingHeadersUpdate(input: { headerId: "gid://gitlab/AuditEvents::Streaming::Header/24255", key: "new-foo", value: "new-bar" }) { + externalAuditEventDestinationDestroy(input: { id: destination }) { errors } } ``` -### Deleting custom HTTP headers +Streaming destination is deleted if: -Group owners can remove a HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID -by [listing all the custom headers](#list-all-custom-headers) on the group. +- The returned `errors` object is empty. +- The API responds with `200 OK`. + +Group owners can remove an HTTP header using the GraphQL `auditEventsStreamingHeadersDestroy` mutation. You can retrieve the header ID +by [listing all the custom HTTP headers](#use-the-api-1) for the group. ```graphql mutation { @@ -233,52 +243,6 @@ mutation { The header is deleted if the returned `errors` object is empty. -### List all custom headers - -List all custom HTTP headers with the API or GitLab UI. - -#### Use the API - -You can list all custom headers for a top-level group as well as their value and ID using the GraphQL `externalAuditEventDestinations` query. The ID -value returned by this query is what you need to pass to the `deletion` mutation. - -```graphql -query { - group(fullPath: "your-group") { - id - externalAuditEventDestinations { - nodes { - destinationUrl - id - headers { - nodes { - key - value - id - } - } - } - } - } -} -``` - -#### Use the GitLab UI - -FLAG: -On self-managed GitLab, by default the UI for this feature is not available. To make it available per group, ask an administrator to -[enable the feature flag](../administration/feature_flags.md) named `custom_headers_streaming_audit_events_ui`. On GitLab.com, the UI for this feature is -not available. The UI for this feature is not ready for production use. - -Users with at least the Owner role for a group can add event streaming destinations and custom HTTP headers for it: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. -1. On the main area, select **Streams** tab. -1. Select **{pencil}** at the right side of an item. -1. A read-only view of the items custom headers is shown. To track progress on adding editing functionality, see the [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). -1. Select **Cancel** to close the read-only view. - ## Verify event authenticity > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345424) in GitLab 14.8. @@ -293,18 +257,41 @@ the destination's value when [listing streaming destinations](#list-streaming-de > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360814) in GitLab 15.2. -Users with at least the Owner role for a group can list event streaming destinations and see the verification tokens: +Users with the Owner role for a group can list streaming destinations and see the verification tokens: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Audit events**. -1. On the main area, select **Streams**. +1. On the main area, select the **Streams**. 1. View the verification token on the right side of each item. +## Payload schema + +> Documentation for an audit event streaming schema was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358149) in GitLab 15.3. + +Streamed audit events have a predictable schema in the body of the response. + +| Field | Description | Notes | +|------------------|------------------------------------------------------------|-----------------------------------------------------------------------------------| +| `author_id` | User ID of the user who triggered the event | | +| `author_name` | Human-readable name of the author that triggered the event | Helpful when the author no longer exists | +| `created_at` | Timestamp when event was triggered | | +| `details` | JSON object containing additional metadata | Has no defined schema but often contains additional information about an event | +| `entity_id` | ID of the audit event's entity | | +| `entity_path` | Full path of the entity affected by the auditable event | | +| `entity_type` | String representation of the type of entity | Acceptable values include `User`, `Group`, and `Key`. This list is not exhaustive | +| `event_type` | String representation of the type of audit event | | +| `id` | Unique identifier for the audit event | Can be used for deduplication if required | +| `ip_address` | IP address of the host used to trigger the event | | +| `target_details` | Additional details about the target | | +| `target_id` | ID of the audit event's target | | +| `target_type` | String representation of the target's type | | + ## Audit event streaming on Git operations > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.0. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/357211) in GitLab 15.1 by default. +> - [Added `details.author_class` field](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. FLAG: On self-managed GitLab, by default this feature is available. To hide the @@ -318,7 +305,7 @@ Streaming audit events can be sent when signed-in users push or pull a project's Audit events are not captured for users that are not signed in. For example, when downloading a public project. -To configure streaming audit events for Git operations, see [Add a new event streaming destination](#add-a-new-event-streaming-destination). +To configure streaming audit events for Git operations, see [Add a new streaming destination](#add-a-new-streaming-destination). ### Headers @@ -346,6 +333,7 @@ Fetch: "entity_type": "Project", "details": { "author_name": "Administrator", + "author_class": "User", "target_id": 29, "target_type": "Project", "target_details": "example-project", @@ -377,6 +365,7 @@ Push: "entity_type": "Project", "details": { "author_name": "Administrator", + "author_class": "User", "target_id": 29, "target_type": "Project", "target_details": "example-project", @@ -398,6 +387,42 @@ Push: } ``` +### Example payloads for SSH events with Deploy Key + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. + +Fetch: + +```json +{ + "id": 1, + "author_id": -3, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "deploy-key-name", + "author_class": "DeployKey", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "deploy-key-name", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-07-26T05:43:53.662Z", + "target_type": "Project", + "target_id": 29, + "event_type": "repository_git_operation" +} +``` + ### Example payloads for HTTP and HTTPS events Fetch: @@ -410,6 +435,7 @@ Fetch: "entity_type": "Project", "details": { "author_name": "Administrator", + "author_class": "User", "target_id": 29, "target_type": "Project", "target_details": "example-project", @@ -441,6 +467,7 @@ Push: "entity_type": "Project", "details": { "author_name": "Administrator", + "author_class": "User", "target_id": 29, "target_type": "Project", "target_details": "example-project", @@ -462,6 +489,40 @@ Push: } ``` +### Example payloads for HTTP and HTTPS events with Deploy Token + +Fetch: + +```json +{ + "id": 1, + "author_id": -2, + "entity_id": 22, + "entity_type": "Project", + "details": { + "author_name": "deploy-token-name", + "author_class": "DeployToken", + "target_id": 22, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "deploy-token-name", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-07-26T05:46:25.850Z", + "target_type": "Project", + "target_id": 22, + "event_type": "repository_git_operation" +} +``` + ### Example payloads for events from GitLab UI download button Fetch: @@ -475,6 +536,7 @@ Fetch: "details": { "custom_message": "Repository Download Started", "author_name": "example_username", + "author_class": "User", "target_id": 29, "target_type": "Project", "target_details": "example-group/example-project", diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index a329adbed22..92504c226fb 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab offers a way to view the changes made within the GitLab server for owners and administrators on a [paid plan](https://about.gitlab.com/pricing/). -GitLab system administrators can also view all audit events by accessing the [`audit_json.log` file](logs.md#audit_jsonlog). +GitLab system administrators can also view all audit events by accessing the [`audit_json.log` file](logs/index.md#audit_jsonlog). The JSON audit log does not include events that are [only streamed](../development/audit_event_guide/index.md#event-streaming). You can: @@ -115,7 +115,7 @@ From there, you can see the following actions: - Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. - Group deploy token was successfully created, revoked, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. - Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- [IP restrictions](../user/group/index.md#group-access-restriction-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. +- [IP restrictions](../user/group/access_and_permissions.md#restrict-group-access-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. - Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge request approvals settings: - Prevent approval by author. @@ -123,6 +123,7 @@ From there, you can see the following actions: - Prevent editing approval rules in projects and merge requests. - Require user password to approve. - Remove all approvals when commits are added to the source branch. +- Changes to streaming audit destination custom HTTP headers. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366350) in GitLab 15.3. Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events) @@ -151,6 +152,7 @@ From there, you can see the following actions: - Added, removed, or updated protected branches - Release was added to a project - Release was updated +- Release was deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94793/) in GitLab 13.5) - Release milestone associations changed - Permission to approve merge requests by committers was updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9) - Permission to approve merge requests by committers was updated. @@ -271,7 +273,7 @@ Don't see the event you want in any of the epics linked above? You can either: - Use the **Audit Event Proposal** issue template to [create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Audit%20Event%20Proposal) to request it. -- [Add it yourself](../development/audit_event_guide/). +- [Add it yourself](../development/audit_event_guide/index.md). ### Removed events diff --git a/doc/administration/audit_reports.md b/doc/administration/audit_reports.md index 6fa592b96db..e33b5153c5b 100644 --- a/doc/administration/audit_reports.md +++ b/doc/administration/audit_reports.md @@ -26,4 +26,4 @@ needs. ## Features - [Audit events](audit_events.md) -- [Log system](logs.md) +- [Log system](logs/index.md) diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index d82683e1778..561aa5d7b2e 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -55,3 +55,9 @@ If you are signed in with auditor access, you: you can push commits or comment on issues. - Can access the same resources using the GitLab UI or API. - Can't view the Admin Area, or perform any administration actions. + +## Maintain auditor users using API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366404) in GitLab 15.3. + +Administrators can use the GitLab API to [create](../api/users.md#user-creation) and [modify](../api/users.md#user-modification) auditor users. diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 8dfa832a103..ced7cdb7119 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -5,7 +5,11 @@ group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Atlassian Crowd OmniAuth Provider **(FREE SELF)** +# Atlassian Crowd OmniAuth provider (deprecated) **(FREE SELF)** + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369117) in GitLab 15.3 and is planned for +removal in 16.0. Authenticate to GitLab using the Atlassian Crowd OmniAuth provider. Enabling this provider also allows Crowd authentication for Git-over-https requests. @@ -68,24 +72,24 @@ this provider also allows Crowd authentication for Git-over-https requests. application_password: 'YOUR_APP_PASSWORD' } } ``` -1. Change `CROWD_SERVER_URL` to the URL of your Crowd server. +1. Change `CROWD_SERVER_URL` to the [base URL of your Crowd server](https://confluence.atlassian.com/crowdkb/how-to-change-the-crowd-base-url-245827278.html). 1. Change `YOUR_APP_NAME` to the application name from Crowd applications page. 1. Change `YOUR_APP_PASSWORD` to the application password you've set. 1. Save the configuration file. -1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../restart_gitlab.md#installations-from-source) for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) (Omnibus GitLab) or [restart](../restart_gitlab.md#installations-from-source) (source installations) for + the changes to take effect. On the sign in page there should now be a Crowd tab in the sign in form. ## Troubleshooting -If you see an error message like the one below when you sign in after Crowd authentication is configured, you may want to consult the Crowd administrator for the Crowd log file to know the exact cause: +### Error: "could not authorize you from Crowd because invalid credentials" -```plaintext -could not authorize you from Crowd because invalid credentials -``` +This error sometimes occurs when a user attempts to authenticate with Crowd. The +Crowd administrator should consult the Crowd log file to know the exact cause of +this error message. -Ensure the Crowd users who need to sign in to GitLab are authorized to the +Ensure the Crowd users who must sign in to GitLab are authorized to the [application](#configure-a-new-crowd-application) in the **Authorization** step. This could be verified by trying "Authentication test" for Crowd (as of 2.11). diff --git a/doc/administration/auth/ldap/img/multi_login.png b/doc/administration/auth/ldap/img/multi_login.png Binary files differdeleted file mode 100644 index 512f403a442..00000000000 --- a/doc/administration/auth/ldap/img/multi_login.png +++ /dev/null diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 05eee338e64..2f0a0db9d6f 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -75,7 +75,7 @@ To configure LDAP integration, add your LDAP server settings in: - `/home/git/gitlab/config/gitlab.yml` for source install instances. After configuring LDAP, to test the configuration, use the -[LDAP check Rake task](../../raketasks/check.md#ldap-check). +[LDAP check Rake task](../../raketasks/ldap.md#check). NOTE: The `encryption` value `simple_tls` corresponds to 'Simple TLS' in the LDAP @@ -95,42 +95,42 @@ This example shows configuration for Omnibus GitLab instances: gitlab_rails['ldap_enabled'] = true gitlab_rails['prevent_ldap_sign_in'] = false gitlab_rails['ldap_servers'] = { -'main' => { - 'label' => 'LDAP', - 'host' => 'ldap.mydomain.com', - 'port' => 389, - 'uid' => 'sAMAccountName', - 'encryption' => 'simple_tls', - 'verify_certificates' => true, - 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', - 'password' => '_the_password_of_the_bind_user', - 'tls_options' => { - 'ca_file' => '', - 'ssl_version' => '', - 'ciphers' => '', - 'cert' => '', - 'key' => '' - }, - 'timeout' => 10, - 'active_directory' => true, - 'allow_username_or_email_login' => false, - 'block_auto_created_users' => false, - 'base' => 'dc=example,dc=com', - 'user_filter' => '', - 'attributes' => { - 'username' => ['uid', 'userid', 'sAMAccountName'], - 'email' => ['mail', 'email', 'userPrincipalName'], - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' - }, - 'lowercase_usernames' => false, - - # EE Only - 'group_base' => '', - 'admin_group' => '', - 'external_groups' => [], - 'sync_ssh_keys' => false + 'main' => { + 'label' => 'LDAP', + 'host' => 'ldap.mydomain.com', + 'port' => 389, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'verify_certificates' => true, + 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', + 'password' => '_the_password_of_the_bind_user', + 'tls_options' => { + 'ca_file' => '', + 'ssl_version' => '', + 'ciphers' => '', + 'cert' => '', + 'key' => '' + }, + 'timeout' => 10, + 'active_directory' => true, + 'allow_username_or_email_login' => false, + 'block_auto_created_users' => false, + 'base' => 'dc=example,dc=com', + 'user_filter' => '', + 'attributes' => { + 'username' => ['uid', 'userid', 'sAMAccountName'], + 'email' => ['mail', 'email', 'userPrincipalName'], + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + }, + 'lowercase_usernames' => false, + + # EE Only + 'group_base' => '', + 'admin_group' => '', + 'external_groups' => [], + 'sync_ssh_keys' => false } } ``` @@ -248,33 +248,34 @@ The following example shows how to configure three LDAP servers in `gitlab.rb`: ```ruby gitlab_rails['ldap_enabled'] = true gitlab_rails['ldap_servers'] = { -'main' => { - 'label' => 'GitLab AD', - 'host' => 'ad.example.org', - 'port' => 636, - ... + 'main' => { + 'label' => 'GitLab AD', + 'host' => 'ad.example.org', + 'port' => 636, + ... }, -'secondary' => { - 'label' => 'GitLab Secondary AD', - 'host' => 'ad-secondary.example.net', - 'port' => 636, - ... + 'secondary' => { + 'label' => 'GitLab Secondary AD', + 'host' => 'ad-secondary.example.net', + 'port' => 636, + ... }, -'tertiary' => { - 'label' => 'GitLab Tertiary AD', - 'host' => 'ad-tertiary.example.net', - 'port' => 636, - ... + 'tertiary' => { + 'label' => 'GitLab Tertiary AD', + 'host' => 'ad-tertiary.example.net', + 'port' => 636, + ... } - } ``` -This example results in the following sign-in page: +This example results in a sign-in page with the following tabs: - +- **GitLab AD**. +- **GitLab Secondary AD**. +- **GitLab Tertiary AD**. ### Set up LDAP user filter @@ -286,9 +287,9 @@ necessary, you can set up an LDAP user filter. The filter must comply with [RFC ```ruby gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'user_filter' => '(employeeType=developer)' + 'main' => { + # snip... + 'user_filter' => '(employeeType=developer)' } } ``` @@ -363,9 +364,9 @@ the configuration option `lowercase_usernames`. By default, this configuration o ```ruby gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'lowercase_usernames' => true + 'main' => { + # snip... + 'lowercase_usernames' => true } } ``` @@ -444,15 +445,15 @@ If initially your LDAP configuration looked like: 1. In `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'bind_dn' => 'admin', - 'password' => '123' - } - } - ``` + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + # snip... + 'bind_dn' => 'admin', + 'password' => '123' + } + } + ``` 1. Edit the encrypted secret: diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index c7572ec0a18..64ef27cbf51 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -181,7 +181,7 @@ The user should now be able to sign in. #### Email has already been taken A user tries to sign in with the correct LDAP credentials, is denied access, -and the [production.log](../../logs.md#productionlog) shows an error that looks like this: +and the [production.log](../../logs/index.md#productionlog) shows an error that looks like this: ```plaintext (LDAP) Error saving user <USER DN> (email@example.com): ["Email has already been taken"] @@ -210,8 +210,8 @@ This shows you which user has this email address. One of two steps must be taken remove this email as a secondary email and make it a primary one so GitLab associates this profile to the LDAP identity. -The user can do either of these steps [in their -profile](../../../user/profile/index.md#access-your-user-profile) or an administrator can do it. +The user can do either of these steps +[in their profile](../../../user/profile/index.md#access-your-user-profile) or an administrator can do it. #### Projects limit errors @@ -426,13 +426,12 @@ Rails.logger.level = Logger::DEBUG LdapAllGroupsSyncWorker.new.perform ``` -Next, [learn how to read the -output](#example-console-output-after-a-group-sync). +Next, [learn how to read the output](#example-console-output-after-a-group-sync). ##### Example console output after a group sync -Like the output from the user sync, the output from the [manual group -sync](#sync-all-groups) is also very verbose. However, it contains lots +Like the output from the user sync, the output from the +[manual group sync](#sync-all-groups) is also very verbose. However, it contains lots of helpful information. Indicates the point where syncing actually begins: @@ -644,10 +643,10 @@ users, [see what to do when no users are found](#no-users-are-found). ### GitLab logs If a user account is blocked or unblocked due to the LDAP configuration, a -message is [logged to `application.log`](../../logs.md#applicationlog). +message is [logged to `application.log`](../../logs/index.md#applicationlog). If there is an unexpected error during an LDAP lookup (configuration error, -timeout), the sign-in is rejected and a message is [logged to `production.log`](../../logs.md#productionlog). +timeout), the sign-in is rejected and a message is [logged to `production.log`](../../logs/index.md#productionlog). ### ldapsearch @@ -743,8 +742,7 @@ For instructions about how to use the rails console, refer to this This provides debug output that shows what GitLab is doing and with what. This value is not persisted, and is only enabled for this session in the Rails console. -To enable debug output in the rails console, [enter the rails -console](#rails-console) and run: +To enable debug output in the rails console, [enter the rails console](#rails-console) and run: ```ruby Rails.logger.level = Logger::DEBUG diff --git a/doc/administration/auth/ldap/ldap_synchronization.md b/doc/administration/auth/ldap/ldap_synchronization.md index b0ada1c11dd..62706a9e3b9 100644 --- a/doc/administration/auth/ldap/ldap_synchronization.md +++ b/doc/administration/auth/ldap/ldap_synchronization.md @@ -128,7 +128,7 @@ To take advantage of group sync, group Owners or users with the [Maintainer role ### Add group links For information on adding group links by using CNs and filters, refer to the -[GitLab groups documentation](../../../user/group/index.md#manage-group-memberships-via-ldap). +[GitLab groups documentation](../../../user/group/access_and_permissions.md#manage-group-memberships-via-ldap). ### Administrator sync diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 60a4cc8706f..8c5bf96e99e 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -250,8 +250,8 @@ but `LocalAccounts` works for authenticating against local, Active Directory acc <OutputClaim ClaimTypeReferenceId="signInNames.emailAddress" PartnerClaimType="email" /> ``` -1. For OIDC discovery to work with B2C, the policy must be configured with an issuer compatible with the [OIDC - specification](https://openid.net/specs/openid-connect-discovery-1_0.html#rfc.section.4.3). +1. For OIDC discovery to work with B2C, the policy must be configured with an issuer compatible with the + [OIDC specification](https://openid.net/specs/openid-connect-discovery-1_0.html#rfc.section.4.3). See the [token compatibility settings](https://docs.microsoft.com/en-us/azure/active-directory-b2c/configure-tokens?pivots=b2c-custom-policy#token-compatibility-settings). In `TrustFrameworkBase.xml` under `JwtIssuer`, set `IssuanceClaimPattern` to `AuthorityWithTfp`: @@ -529,8 +529,7 @@ If you're having trouble, here are some tips: 1. Check your system clock to ensure the time is synchronized properly. -1. As mentioned in [the - documentation](https://github.com/m0n9oose/omniauth_openid_connect), +1. As mentioned in [the documentation](https://github.com/m0n9oose/omniauth_openid_connect), make sure `issuer` corresponds to the base URL of the Discovery URL. For example, `https://accounts.google.com` is used for the URL `https://accounts.google.com/.well-known/openid-configuration`. @@ -540,5 +539,4 @@ If you're having trouble, here are some tips: If you are seeing 401 errors upon retrieving the `userinfo` endpoint, you may want to check your OpenID Web server configuration. For example, for [`oauth2-server-php`](https://github.com/bshaffer/oauth2-server-php), you - may need to [add a configuration parameter to - Apache](https://github.com/bshaffer/oauth2-server-php/issues/926#issuecomment-387502778). + may need to [add a configuration parameter to Apache](https://github.com/bshaffer/oauth2-server-php/issues/926#issuecomment-387502778). diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 869de929eb8..8c7f8bf766d 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -97,7 +97,7 @@ These features can also help with compliance requirements: projects): Search dependencies for their licenses. This lets you determine if the licenses of your project's dependencies are compatible with your project's license. -- [**Lock project membership to group**](../user/group/index.md#prevent-members-from-being-added-to-projects-in-a-group) +- [**Lock project membership to group**](../user/group/access_and_permissions.md#prevent-members-from-being-added-to-projects-in-a-group) (for groups): Group owners can prevent new members from being added to projects within a group. - [**LDAP group sync**](auth/ldap/ldap_synchronization.md#group-sync) (for diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index 06eb9ffc84e..99876cdf503 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -46,6 +46,7 @@ required number of seconds. "user", "pipeline", "builds", + "total_builds_count", "namespace" ], "properties" : { @@ -61,7 +62,9 @@ required number of seconds. "properties": { "id": { "type": "integer" }, "path": { "type": "string" }, - "created_at": { "type": ["string", "null"], "format": "date-time" } + "created_at": { "type": ["string", "null"], "format": "date-time" }, + "shared_runners_enabled": { "type": "boolean" }, + "group_runners_enabled": { "type": "boolean" } } }, "user": { @@ -121,6 +124,7 @@ required number of seconds. } } }, + "total_builds_count": { "type": "integer" }, "namespace": { "type": "object", "required": [ diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index 833b9a877e9..a2d4f35a7c3 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -41,8 +41,8 @@ To bring the former **primary** site up to date: NOTE: If you [changed the DNS records](index.md#step-4-optional-updating-the-primary-domain-dns-record) - for this site during disaster recovery procedure you may need to [block - all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-site) + for this site during disaster recovery procedure you may need to + [block all the writes to this site](planned_failover.md#prevent-updates-to-the-primary-site) during this procedure. 1. [Set up database replication](../setup/database.md). In this case, the **secondary** site diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index f457cb4b0a2..0cae94fcec1 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -572,7 +572,7 @@ and after that you also need two extra steps. postgresql['md5_auth_cidr_addresses'] = ['<primary_site_ip>/32', '<secondary_site_ip>/32'] # Every secondary site needs to have its own slot so specify the number of secondary sites you're going to have - postgresql['max_replication_slots'] = 1 + # postgresql['max_replication_slots'] = 1 # Set this to be the number of Geo secondary nodes if you have more than one ## ## Disable automatic database migrations temporarily @@ -581,7 +581,7 @@ and after that you also need two extra steps. gitlab_rails['auto_migrate'] = false ``` - (For more details about these settings you can read [Configure the primary server](../setup/database.md#step-1-configure-the-primary-server)) + (For more details about these settings you can read [Configure the primary server](../setup/database.md#step-1-configure-the-primary-site)) 1. Save the file and reconfigure GitLab for the database listen changes and the replication slot changes to be applied: diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index 5a5d896c20a..6c3353e7d7e 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -45,7 +45,7 @@ be adapted for use with any other file-based data, such as [GitLab Pages](../../ ### Container registry By default, the container registry is not automatically replicated to secondary -sites and this needs to be manually configured, see [Docker Registry for a secondary site](../replication/docker_registry.md). +sites and this needs to be manually configured, see [Container Registry for a secondary site](../replication/container_registry.md). If you are using local storage on your current primary site for the container registry, you can `rsync` the container registry objects to the secondary diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index cf7d2649142..e3bf5ccdfe7 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -119,8 +119,8 @@ The following are required to run Geo: The following operating systems are known to ship with a current version of OpenSSH: - [CentOS](https://www.centos.org) 7.4 or later - [Ubuntu](https://ubuntu.com) 16.04 or later -- PostgreSQL 12 with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) - - PostgreSQL 13 is not supported for Geo, see [epic 3832](https://gitlab.com/groups/gitlab-org/-/epics/3832) +- PostgreSQL 12 or 13 with [Streaming Replication](https://wiki.postgresql.org/wiki/Streaming_Replication) + - Note,[PostgreSQL 12 is deprecated](../../update/deprecations.md#postgresql-12-deprecated) and will be removed in GitLab 16.0. - Git 2.9 or later - Git-lfs 2.4.2 or later on the user side when using LFS - All sites must run [the same GitLab and PostgreSQL versions](setup/database.md#postgresql-replication). @@ -283,7 +283,7 @@ For information on using Geo in disaster recovery situations to mitigate data-lo ### Replicating the Container Registry -For more information on how to replicate the Container Registry, see [Docker Registry for a **secondary** site](replication/docker_registry.md). +For more information on how to replicate the Container Registry, see [Container Registry for a **secondary** site](replication/container_registry.md). ### Geo secondary proxy diff --git a/doc/administration/geo/replication/container_registry.md b/doc/administration/geo/replication/container_registry.md new file mode 100644 index 00000000000..b425e5dcc0d --- /dev/null +++ b/doc/administration/geo/replication/container_registry.md @@ -0,0 +1,167 @@ +--- +stage: Systems +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: howto +--- + +# Container Registry for a secondary site **(PREMIUM SELF)** + +You can set up a Container Registry on your **secondary** Geo site that mirrors the one on the **primary** Geo site. + +## Supported container registries + +Geo supports the following types of container registries: + +- [Docker](https://docs.docker.com/registry/) +- [OCI](https://github.com/opencontainers/distribution-spec/blob/main/spec.md) + +## Supported image formats + +The following container image formats are support by Geo: + +- [Docker V2, schema 1](https://docs.docker.com/registry/spec/manifest-v2-1/) +- [Docker V2, schema 2](https://docs.docker.com/registry/spec/manifest-v2-2/) +- [OCI (Open Container Initiative)](https://github.com/opencontainers/image-spec) + +In addition, Geo also supports [BuildKit cache images](https://github.com/moby/buildkit). + +## Supported storage + +### Docker + +For more information on supported registry storage drivers see +[Docker registry storage drivers](https://docs.docker.com/registry/storage-drivers/) + +Read the [Load balancing considerations](https://docs.docker.com/registry/deploying/#load-balancing-considerations) +when deploying the Registry, and how to set up the storage driver for the GitLab integrated +[Container Registry](../../packages/container_registry.md#use-object-storage). + +### Registries that support OCI artifacts + +The following registries support OCI artifacts: + +- CNCF Distribution - local/offline verification +- Azure Container Registry (ACR) +- Amazon Elastic Container Registry (ECR) +- Google Artifact Registry (GAR) +- GitHub Packages container registry (GHCR) +- Bundle Bar + +For more information, see the [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec). + +## Configure Container Registry replication + +You can enable a storage-agnostic replication so it +can be used for cloud or local storage. Whenever a new image is pushed to the +**primary** site, each **secondary** site pulls it to its own container +repository. + +To configure Container Registry replication: + +1. Configure the [**primary** site](#configure-primary-site). +1. Configure the [**secondary** site](#configure-secondary-site). +1. Verify Container Registry [replication](#verify-replication). + +### Configure **primary** site + +Make sure that you have Container Registry set up and working on +the **primary** site before following the next steps. + +We need to make Container Registry send notification events to the +**primary** site. + +1. SSH into your GitLab **primary** server and login as root: + + ```shell + sudo -i + ``` + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + registry['notifications'] = [ + { + 'name' => 'geo_event', + 'url' => 'https://<example.com>/api/v4/container_registry_event/events', + 'timeout' => '500ms', + 'threshold' => 5, + 'backoff' => '1s', + 'headers' => { + 'Authorization' => ['<replace_with_a_secret_token>'] + } + } + ] + ``` + + NOTE: + Replace `<example.com>` with the `external_url` defined in your primary site's `/etc/gitlab/gitlab.rb` file, and + replace `<replace_with_a_secret_token>` with a case sensitive alphanumeric string + that starts with a letter. You can generate one with `< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c 32 | sed "s/^[0-9]*//"; echo` + + NOTE: + If you use an external Registry (not the one integrated with GitLab), you must add + these settings to its configuration yourself. In this case, you also have to specify + notification secret in `registry.notification_secret` section of + `/etc/gitlab/gitlab.rb` file. + + NOTE: + If you use GitLab HA, you also have to specify + the notification secret in `registry.notification_secret` section of + `/etc/gitlab/gitlab.rb` file for every web node. + +1. Reconfigure the **primary** node for the change to take effect: + + ```shell + gitlab-ctl reconfigure + ``` + +### Configure **secondary** site + +Make sure you have Container Registry set up and working on +the **secondary** site before following the next steps. + +The following steps should be done on each **secondary** site you're +expecting to see the container images replicated. + +Because we need to allow the **secondary** site to communicate securely with +the **primary** site Container Registry, we need to have a single key +pair for all the sites. The **secondary** site uses this key to +generate a short-lived JWT that is pull-only-capable to access the +**primary** site Container Registry. + +For each application and Sidekiq node on the **secondary** site: + +1. SSH into the node and login as the `root` user: + + ```shell + sudo -i + ``` + +1. Copy `/var/opt/gitlab/gitlab-rails/etc/gitlab-registry.key` from the **primary** to the node. + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + gitlab_rails['geo_registry_replication_enabled'] = true + + # Primary registry's hostname and port, it will be used by + # the secondary node to directly communicate to primary registry + gitlab_rails['geo_registry_replication_primary_api_url'] = 'https://primary.example.com:5050/' + ``` + +1. Reconfigure the node for the change to take effect: + + ```shell + gitlab-ctl reconfigure + ``` + +### Verify replication + +To verify Container Registry replication is working, on the **secondary** site: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Geo > Nodes**. + The initial replication, or "backfill", is probably still in progress. + +You can monitor the synchronization process on each Geo site from the **primary** site's **Geo Nodes** dashboard in your browser. diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index acd27d5feed..40b71684bac 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -57,6 +57,8 @@ verification methods: | Blobs | Pipeline artifacts _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | Pages _(file system)_ | Geo with API | SHA256 checksum | | Blobs | Pages _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | CI Secure Files _(file system)_ | Geo with API | SHA256 checksum | +| Blobs | CI Secure Files _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | - (*1*): Redis replication can be used as part of HA with Redis sentinel. It's not used between Geo sites. - (*2*): Object storage replication can be performed by Geo or by your object storage provider/appliance @@ -185,8 +187,8 @@ successfully, you must replicate their data using some other means. |Feature | Replicated (added in GitLab version) | Verified (added in GitLab version) | GitLab-managed object storage replication (added in GitLab version) | GitLab-managed object storage verification (added in GitLab version) | Notes | |:--------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------|:---------------------------------------------------------------------------|:--------------------------------------------------------------------|:----------------------------------------------------------------|:------| |[Application data in PostgreSQL](../../postgresql/index.md) | **Yes** (10.2) | **Yes** (10.2) | N/A | N/A | | -|[Project repository](../../../user/project/repository/) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | -|[Project wiki repository](../../../user/project/wiki/) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | +|[Project repository](../../../user/project/repository/index.md) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | +|[Project wiki repository](../../../user/project/wiki/index.md) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | |[Group wiki repository](../../../user/project/wiki/group.md) | [**Yes** (13.10)](https://gitlab.com/gitlab-org/gitlab/-/issues/208147) | No | N/A | N/A | Behind feature flag `geo_group_wiki_repository_replication`, enabled by default. | |[Uploads](../../uploads.md) | **Yes** (10.2) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Replication is behind the feature flag `geo_upload_replication`, enabled by default. Verification was behind the feature flag `geo_upload_verification`, removed in 14.8. | |[LFS objects](../../lfs/index.md) | **Yes** (10.2) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696).<br /><br />Replication is behind the feature flag `geo_lfs_object_replication`, enabled by default. Verification was behind the feature flag `geo_lfs_object_verification`, removed in 14.7. | @@ -194,7 +196,8 @@ successfully, you must replicate their data using some other means. |[Project snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | N/A | N/A | | |[CI job artifacts](../../../ci/pipelines/job_artifacts.md) | **Yes** (10.4) | **Yes** (14.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_job_artifact_replication`, enabled by default in 14.10. | |[CI Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/pipeline_artifact.rb) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Persists additional artifacts after a pipeline completes. | -|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | No | No | Disabled by default. See [instructions](docker_registry.md) to enable. | +|[CI Secure Files](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/secure_file.rb) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_ci_secure_file_replication`, enabled by default in 15.3. | +|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3) | No | No | No | Disabled by default. See [instructions](container_registry.md) to enable. | |[Infrastructure Registry](../../../user/packages/infrastructure_registry/index.md) | **Yes** (14.0) | **Yes** (14.0) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | |[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | N/A | N/A | Designs also require replication of LFS objects and Uploads. | |[Package Registry](../../../user/packages/package_registry/index.md) | **Yes** (13.2) | **Yes** (13.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | @@ -202,9 +205,10 @@ successfully, you must replicate their data using some other means. |[External merge request diffs](../../merge_request_diffs.md) | **Yes** (13.5) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Replication is behind the feature flag `geo_merge_request_diff_replication`, enabled by default. Verification was behind the feature flag `geo_merge_request_diff_verification`, removed in 14.7.| |[Versioned snippets](../../../user/snippets.md#versioned-snippets) | [**Yes** (13.7)](https://gitlab.com/groups/gitlab-org/-/epics/2809) | [**Yes** (14.2)](https://gitlab.com/groups/gitlab-org/-/epics/2810) | N/A | N/A | Verification was implemented behind the feature flag `geo_snippet_repository_verification` in 13.11, and the feature flag was removed in 14.2. | |[GitLab Pages](../../pages/index.md) | [**Yes** (14.3)](https://gitlab.com/groups/gitlab-org/-/epics/589) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_pages_deployment_replication`, enabled by default. Verification was behind the feature flag `geo_pages_deployment_verification`, removed in 14.7. | +|[Project-level Secure files](../../../ci/secure_files/index.md) | **Yes** (15.3) | **Yes** (15.3) | **Yes** (15.3) | [No](object_storage.md#verification-of-files-in-object-storage) | | |[Incident Metric Images](../../../operations/incident_management/incidents.md#metrics) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362561) | No | No | | |[Alert Metric Images](../../../operations/incident_management/alerts.md#metrics-tab) | [Planned](https://gitlab.com/gitlab-org/gitlab/-/issues/362564) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/362564) | No | No | | |[Server-side Git hooks](../../server_hooks.md) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/1867) | No | N/A | N/A | Not planned because of current implementation complexity, low customer interest, and availability of alternatives to hooks. | |[Elasticsearch integration](../../../integration/elasticsearch.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/1186) | No | No | No | Not planned because further product discovery is required and Elasticsearch (ES) clusters can be rebuilt. Secondaries use the same ES cluster as the primary. | |[Dependency proxy images](../../../user/packages/dependency_proxy/index.md) | [Not planned](https://gitlab.com/gitlab-org/gitlab/-/issues/259694) | No | No | No | Blocked by [Geo: Secondary Mimicry](https://gitlab.com/groups/gitlab-org/-/epics/1528). Replication of this cache is not needed for disaster recovery purposes because it can be recreated from external sources. | -|[Vulnerability Export](../../../user/application_security/vulnerability_report/#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | +|[Vulnerability Export](../../../user/application_security/vulnerability_report/index.md#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | diff --git a/doc/administration/geo/replication/disable_geo.md b/doc/administration/geo/replication/disable_geo.md index 5d4ae12c990..f0658ae45a2 100644 --- a/doc/administration/geo/replication/disable_geo.md +++ b/doc/administration/geo/replication/disable_geo.md @@ -28,7 +28,7 @@ To disable Geo, you need to first remove all your secondary Geo sites, which mea anymore on these sites. You can follow our docs to [remove your secondary Geo sites](remove_geo_site.md). If the current site that you want to keep using is a secondary site, you need to first promote it to primary. -You can use our steps on [how to promote a secondary site](../disaster_recovery/#step-3-promoting-a-secondary-site) +You can use our steps on [how to promote a secondary site](../disaster_recovery/index.md#step-3-promoting-a-secondary-site) to do that. ## Remove the primary site from the UI diff --git a/doc/administration/geo/replication/docker_registry.md b/doc/administration/geo/replication/docker_registry.md index 855e33d9a51..d0af6f2a66f 100644 --- a/doc/administration/geo/replication/docker_registry.md +++ b/doc/administration/geo/replication/docker_registry.md @@ -1,137 +1,11 @@ --- -stage: Systems -group: Geo -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: howto +redirect_to: 'container_registry.md' +remove_date: '2022-10-29' --- -# Docker Registry for a secondary site **(PREMIUM SELF)** +This document was moved to [another location](container_registry.md). -You can set up a [Docker Registry](https://docs.docker.com/registry/) on your -**secondary** Geo site that mirrors the one on the **primary** Geo site. - -## Storage support - -Docker Registry currently supports a few types of storage. If you choose a -distributed storage (`azure`, `gcs`, `s3`, `swift`, or `oss`) for your Docker -Registry on the **primary** site, you can use the same storage for a **secondary** -Docker Registry as well. For more information, read the -[Load balancing considerations](https://docs.docker.com/registry/deploying/#load-balancing-considerations) -when deploying the Registry, and how to set up the storage driver for the GitLab -integrated [Container Registry](../../packages/container_registry.md#use-object-storage). - -## Replicating Docker Registry - -You can enable a storage-agnostic replication so it -can be used for cloud or local storage. Whenever a new image is pushed to the -**primary** site, each **secondary** site pulls it to its own container -repository. - -To configure Docker Registry replication: - -1. Configure the [**primary** site](#configure-primary-site). -1. Configure the [**secondary** site](#configure-secondary-site). -1. Verify Docker Registry [replication](#verify-replication). - -### Configure **primary** site - -Make sure that you have Container Registry set up and working on -the **primary** site before following the next steps. - -We need to make Docker Registry send notification events to the -**primary** site. - -1. SSH into your GitLab **primary** server and login as root: - - ```shell - sudo -i - ``` - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - registry['notifications'] = [ - { - 'name' => 'geo_event', - 'url' => 'https://<example.com>/api/v4/container_registry_event/events', - 'timeout' => '500ms', - 'threshold' => 5, - 'backoff' => '1s', - 'headers' => { - 'Authorization' => ['<replace_with_a_secret_token>'] - } - } - ] - ``` - - NOTE: - Replace `<example.com>` with the `external_url` defined in your primary site's `/etc/gitlab/gitlab.rb` file, and - replace `<replace_with_a_secret_token>` with a case sensitive alphanumeric string - that starts with a letter. You can generate one with `< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c 32 | sed "s/^[0-9]*//"; echo` - - NOTE: - If you use an external Registry (not the one integrated with GitLab), you must add - these settings to its configuration yourself. In this case, you also have to specify - notification secret in `registry.notification_secret` section of - `/etc/gitlab/gitlab.rb` file. - - NOTE: - If you use GitLab HA, you also have to specify - the notification secret in `registry.notification_secret` section of - `/etc/gitlab/gitlab.rb` file for every web node. - -1. Reconfigure the **primary** node for the change to take effect: - - ```shell - gitlab-ctl reconfigure - ``` - -### Configure **secondary** site - -Make sure you have Container Registry set up and working on -the **secondary** site before following the next steps. - -The following steps should be done on each **secondary** site you're -expecting to see the Docker images replicated. - -Because we need to allow the **secondary** site to communicate securely with -the **primary** site Container Registry, we need to have a single key -pair for all the sites. The **secondary** site uses this key to -generate a short-lived JWT that is pull-only-capable to access the -**primary** site Container Registry. - -For each application and Sidekiq node on the **secondary** site: - -1. SSH into the node and login as the `root` user: - - ```shell - sudo -i - ``` - -1. Copy `/var/opt/gitlab/gitlab-rails/etc/gitlab-registry.key` from the **primary** to the node. - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - gitlab_rails['geo_registry_replication_enabled'] = true - - # Primary registry's hostname and port, it will be used by - # the secondary node to directly communicate to primary registry - gitlab_rails['geo_registry_replication_primary_api_url'] = 'https://primary.example.com:5050/' - ``` - -1. Reconfigure the node for the change to take effect: - - ```shell - gitlab-ctl reconfigure - ``` - -### Verify replication - -To verify Container Registry replication is working, on the **secondary** site: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Geo > Nodes**. - The initial replication, or "backfill", is probably still in progress. - -You can monitor the synchronization process on each Geo site from the **primary** site's **Geo Nodes** dashboard in your browser. +<!-- This redirect file can be deleted after <2022-10-29>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/administration/geo/replication/faq.md b/doc/administration/geo/replication/faq.md index bdf1771e8a8..7a67af1cfa2 100644 --- a/doc/administration/geo/replication/faq.md +++ b/doc/administration/geo/replication/faq.md @@ -65,10 +65,10 @@ connectivity between your sites, your hardware, and so on. That's totally fine. We use HTTP(s) to fetch repository changes from the **primary** site to all **secondary** sites. -## Is this possible to set up a Docker Registry for a **secondary** site that mirrors the one on the **primary** site? +## Is this possible to set up a Container Registry for a **secondary** site that mirrors the one on the **primary** site? -Yes. See [Docker Registry for a **secondary** site](docker_registry.md). +Yes. See [Container Registry for a **secondary** site](container_registry.md). -## Can you login to a secondary site? +## Can you log in to a secondary site? Yes, but secondary sites receive all authentication data (like user accounts and logins) from the primary instance. This means you are re-directed to the primary for authentication and then routed back. diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 082ecbbb208..26d192f62cd 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -129,7 +129,7 @@ http://secondary.example.com/ ``` To find more details about failed items, check -[the `gitlab-rails/geo.log` file](../../troubleshooting/log_parsing.md#find-most-common-geo-sync-errors) +[the `gitlab-rails/geo.log` file](../../logs/log_parsing.md#find-most-common-geo-sync-errors) ### Check if PostgreSQL replication is working @@ -191,7 +191,7 @@ If a replication slot is inactive, the `pg_wal` logs corresponding to the slot are reserved forever (or until the slot is active again). This causes continuous disk usage growth and the following messages appear repeatedly in the -[PostgreSQL logs](../../logs.md#postgresql-logs): +[PostgreSQL logs](../../logs/index.md#postgresql-logs): ```plaintext WARNING: oldest xmin is far in the past @@ -331,8 +331,7 @@ Be sure to restart PostgreSQL for this to take effect. See the This occurs when PostgreSQL does not have a replication slot for the **secondary** node by that name. -You may want to rerun the [replication -process](../setup/database.md) on the **secondary** node . +You may want to rerun the [replication process](../setup/database.md) on the **secondary** node . ### Message: "Command exceeded allowed execution time" when setting up replication? @@ -376,7 +375,7 @@ log data to build up in `pg_xlog`. Removing the unused slots can reduce the amou Slots where `active` is `f` are not active. - When this slot should be active, because you have a **secondary** node configured using that slot, - sign in to that **secondary** node and check the [PostgreSQL logs](../../logs.md#postgresql-logs) + sign in to that **secondary** node and check the [PostgreSQL logs](../../logs/index.md#postgresql-logs) to view why the replication is not running. - If you are no longer using the slot (for example, you no longer have Geo enabled), you can remove it with in the @@ -510,7 +509,7 @@ To solve this: 1. Back up [the `.git` folder](../../repository_storage_types.md#translate-hashed-storage-paths). -1. Optional. [Spot-check](../../troubleshooting/log_parsing.md#find-all-projects-affected-by-a-fatal-git-problem) +1. Optional. [Spot-check](../../logs/log_parsing.md#find-all-projects-affected-by-a-fatal-git-problem) a few of those IDs whether they indeed correspond to a project with known Geo replication failures. Use `fatal: 'geo'` as the `grep` term and the following API call: @@ -597,7 +596,7 @@ to start again from scratch, there are a few steps that can help you: gitlab-ctl stop geo-logcursor ``` - You can watch the [Sidekiq logs](../../logs.md#sidekiq-logs) to know when Sidekiq jobs processing has finished: + You can watch the [Sidekiq logs](../../logs/index.md#sidekiq-logs) to know when Sidekiq jobs processing has finished: ```shell gitlab-ctl tail sidekiq @@ -837,7 +836,7 @@ to transfer each affected repository from the primary to the secondary site. The following are possible error messages that might be encountered during failover or when promoting a secondary to a primary node with strategies to resolve them. -### Message: ActiveRecord::RecordInvalid: Validation failed: Name has already been taken +### Message: `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` When [promoting a **secondary** site](../disaster_recovery/index.md#step-3-promoting-a-secondary-site), you might encounter the following error message: @@ -869,11 +868,10 @@ or `gitlab-ctl promote-to-primary-node`, either: ``` - Upgrade to GitLab 12.6.3 or later if it is safe to do so. For example, - if the failover was just a test. A [caching-related - bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was - fixed. + if the failover was just a test. A + [caching-related bug](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22021) was fixed. -### Message: ActiveRecord::RecordInvalid: Validation failed: Enabled Geo primary node cannot be disabled +### Message: `ActiveRecord::RecordInvalid: Validation failed: Enabled Geo primary node cannot be disabled` If you disabled a secondary node, either with the [replication pause task](../index.md#pausing-and-resuming-replication) (GitLab 13.2) or by using the user interface (GitLab 13.1 and earlier), you must first @@ -1127,12 +1125,6 @@ Geo secondary sites continue to replicate and verify data, and the secondary sit This bug was [fixed in GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/292983). -### GitLab Pages return 404 errors after promoting - -This is due to [Pages data not being managed by Geo](datatypes.md#limitations-on-replicationverification). -Find advice to resolve those error messages in the -[Pages administration documentation](../../../administration/pages/index.md#404-error-after-promoting-a-geo-secondary-to-a-primary-node). - ### Primary site returns 500 error when accessing `/admin/geo/replication/projects` Navigating to **Admin > Geo > Replication** (or `/admin/geo/replication/projects`) on a primary Geo site, shows a 500 error, while that same link on the secondary works fine. The primary's `production.log` has a similar entry to the following: @@ -1146,7 +1138,28 @@ Geo::TrackingBase::SecondaryNotConfigured: Geo secondary database is not configu On a Geo primary site this error can be ignored. -This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). +This happens because GitLab is attempting to display registries from the [Geo tracking database](../../../administration/geo/index.md#geo-tracking-database) which doesn't exist on the primary site (only the original projects exist on the primary; no replicated projects are present, therefore no tracking database exists). + +### Secondary site returns 400 error "Request header or cookie too large" + +This error can happen when the internal URL of the primary site is incorrect. + +For example, when you use a unified URL and the primary site's internal URL is also equal to the external URL. This causes a loop when a secondary site proxies requests to the primary site's internal URL. + +To fix this issue, set the primary site's internal URL to a URL that is: + +- Unique to the primary site. +- Accessible from all secondary sites. + +1. Enter the [Rails console](../../operations/rails_console.md) on the primary site. + +1. Run the following, replacing `https://unique.url.for.primary.site` with your specific internal URL. + For example, depending on your network configuration, you could use an IP address, like + `http://1.2.3.4`. + + ```ruby + GeoNode.where(primary: true).first.update!(internal_url: "https://unique.url.for.primary.site") + ``` ## Fixing client errors @@ -1158,6 +1171,23 @@ requests redirected from the secondary to the primary node do not properly send Authorization header. This may result in either an infinite `Authorization <-> Redirect` loop, or Authorization error messages. +### Error: Net::ReadTimeout when pushing through SSH on a Geo secondary + +When you push large repositories through SSH on a Geo secondary site, you may encounter a timeout. +This is because Rails proxies the push to the primary and has a 60 second default timeout, +[as described in this Geo issue](https://gitlab.com/gitlab-org/gitlab/-/issues/7405). + +Current workarounds are: + +- Push through HTTP instead, where Workhorse proxies the request to the primary (or redirects to the primary if Geo proxying is not enabled). +- Push directly to the primary. + +Example log (`gitlab-shell.log`): + +```plaintext +Failed to contact primary https://primary.domain.com/namespace/push_test.git\\nError: Net::ReadTimeout\",\"result\":null}" code=500 method=POST pid=5483 url="http://127.0.0.1:3000/api/v4/geo/proxy_git_push_ssh/push" +``` + ## Recovering from a partial failover The partial failover to a secondary Geo *site* may be the result of a temporary/transient issue. Therefore, first attempt to run the promote command again. diff --git a/doc/administration/geo/replication/tuning.md b/doc/administration/geo/replication/tuning.md index 670459624f3..755ab45a76c 100644 --- a/doc/administration/geo/replication/tuning.md +++ b/doc/administration/geo/replication/tuning.md @@ -30,7 +30,7 @@ However, this may not lead to more downloads in parallel unless the number of available Sidekiq threads is also increased. For example, if repository synchronization concurrency is increased from 25 to 50, you may also want to increase the number of Sidekiq threads from 25 to 50. See the -[Sidekiq concurrency documentation](../../operations/extra_sidekiq_processes.md#number-of-threads) +[Sidekiq concurrency documentation](../../sidekiq/extra_sidekiq_processes.md#number-of-threads) for more details. ## Repository re-verification diff --git a/doc/administration/geo/replication/upgrading_the_geo_sites.md b/doc/administration/geo/replication/upgrading_the_geo_sites.md index 30961de0381..ce1ff4fe6a5 100644 --- a/doc/administration/geo/replication/upgrading_the_geo_sites.md +++ b/doc/administration/geo/replication/upgrading_the_geo_sites.md @@ -28,12 +28,23 @@ and cause downtime. If you want to avoid downtime, consider using To upgrade the Geo sites when a new GitLab version is released, upgrade **primary** and all **secondary** sites: -1. **Optional:** [Pause replication on each **secondary** sites.](../index.md#pausing-and-resuming-replication) +1. Optional. [Pause replication on each **secondary** site](../index.md#pausing-and-resuming-replication) + to protect the disaster recovery (DR) capability of the **secondary** sites. 1. SSH into each node of the **primary** site. 1. [Upgrade GitLab on the **primary** site](../../../update/package/index.md#upgrade-using-the-official-repositories). +1. Perform testing on the **primary** site, particularly if you paused replication in step 1 to protect DR. [There are some suggestions for post-upgrade testing](../../../update/plan_your_upgrade.md#pre-upgrade-and-post-upgrade-checks) in the upgrade documentation. 1. SSH into each node of **secondary** sites. 1. [Upgrade GitLab on each **secondary** site](../../../update/package/index.md#upgrade-using-the-official-repositories). -1. If you paused replication in step 1, [resume replication on each **secondary**](../index.md#pausing-and-resuming-replication) +1. If you paused replication in step 1, [resume replication on each **secondary**](../index.md#pausing-and-resuming-replication). + Then, restart Puma and Sidekiq on each **secondary** site. This is to ensure they + are initialized against the newer database schema that is now replicated from + the previously upgraded **primary** site. + + ```shell + sudo gitlab-ctl restart sidekiq + sudo gitlab-ctl restart puma + ``` + 1. [Test](#check-status-after-upgrading) **primary** and **secondary** sites, and check version in each. ### Check status after upgrading diff --git a/doc/administration/geo/replication/version_specific_upgrades.md b/doc/administration/geo/replication/version_specific_upgrades.md index f0925bdf87e..350310c7076 100644 --- a/doc/administration/geo/replication/version_specific_upgrades.md +++ b/doc/administration/geo/replication/version_specific_upgrades.md @@ -178,11 +178,15 @@ GitLab 13.9 through GitLab 14.3 are affected by a bug in which enabling [GitLab ## Upgrading to GitLab 13.7 -We've detected an issue with the `FetchRemove` call used by Geo secondaries. -This causes performance issues as we execute reference transaction hooks for -each upgraded reference. Delay any upgrade attempts until this is in the -[13.7.5 patch release.](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3002). -More details are available [in this issue](https://gitlab.com/gitlab-org/git/-/issues/79). +- We've detected an issue with the `FetchRemove` call used by Geo secondaries. + This causes performance issues as we execute reference transaction hooks for + each upgraded reference. Delay any upgrade attempts until this is in the + [13.7.5 patch release.](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3002). + More details are available [in this issue](https://gitlab.com/gitlab-org/git/-/issues/79). +- A new secret is generated in `/etc/gitlab/gitlab-secrets.json`. + In an HA GitLab or GitLab Geo environment, secrets need to be the same on all nodes. + Ensure this new secret is also accounted for if you are manually syncing the file across + nodes, or manually specifying secrets in `/etc/gitlab/gitlab.rb`. ## Upgrading to GitLab 13.5 @@ -243,8 +247,8 @@ the recommended procedure, see the ## Upgrading to GitLab 12.9 WARNING: -GitLab 12.9.0 through GitLab 12.9.3 are affected by [a bug that stops -repository verification](https://gitlab.com/gitlab-org/gitlab/-/issues/213523). +GitLab 12.9.0 through GitLab 12.9.3 are affected by +[a bug that stops repository verification](https://gitlab.com/gitlab-org/gitlab/-/issues/213523). The issue is fixed in GitLab 12.9.4. Upgrade to GitLab 12.9.4 or later. By default, GitLab 12.9 attempts to upgrade the embedded PostgreSQL server @@ -397,6 +401,6 @@ For the recommended procedure, see the ## Upgrading to GitLab 12.0 WARNING: -This version is affected by a [bug that results in new LFS objects not being -replicated to Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). +This version is affected by a +[bug that results in new LFS objects not being replicated to Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/32696). The issue is fixed in GitLab 12.1. Be sure to upgrade to GitLab 12.1 or later. diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index e8c290e197b..6c1812b2754 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -112,8 +112,9 @@ gitlab: Since GitLab 15.1, Geo secondary proxying is enabled by default for separate URLs also. -There are minor known issues linked in the ["Geo secondary proxying with separate URLs" -epic](https://gitlab.com/groups/gitlab-org/-/epics/6865). You can also add feedback in the epic about any use-cases that +There are minor known issues linked in the +["Geo secondary proxying with separate URLs" epic](https://gitlab.com/groups/gitlab-org/-/epics/6865). +You can also add feedback in the epic about any use-cases that are not possible anymore with proxying enabled. If you run into issues, to disable this feature, disable the `geo_secondary_proxy_separate_urls` feature flag. diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index c0ed7829fce..8a919a0a269 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -8,7 +8,7 @@ type: howto # Geo database replication **(PREMIUM SELF)** This document describes the minimal required steps to replicate your primary -GitLab database to a secondary node's database. You may have to change some +GitLab database to a secondary site's database. You may have to change some values, based on attributes including your database's setup and size. NOTE: @@ -41,31 +41,31 @@ instructions on setting up replication with a Patroni cluster. ### PostgreSQL replication -The GitLab **primary** node where the write operations happen connects to -the **primary** database server, and **secondary** nodes +The GitLab **primary** site where the write operations happen connects to +the **primary** database server, and **secondary** sites connect to their own database servers (which are read-only). We recommend using [PostgreSQL replication slots](https://medium.com/@tk512/replication-slots-in-postgresql-b4b03d277c75) -to ensure that the **primary** node retains all the data necessary for the **secondary** nodes to +to ensure that the **primary** site retains all the data necessary for the **secondary** sites to recover. See below for more details. The following guide assumes that: - You are using Omnibus and therefore you are using PostgreSQL 12 or later which includes the [`pg_basebackup` tool](https://www.postgresql.org/docs/12/app-pgbasebackup.html). -- You have a **primary** node already set up (the GitLab server you are +- You have a **primary** site already set up (the GitLab server you are replicating from), running Omnibus' PostgreSQL (or equivalent version), and - you have a new **secondary** server set up with the same + you have a new **secondary** site set up with the same [versions of PostgreSQL](../index.md#requirements-for-running-geo), - OS, and GitLab on all nodes. + OS, and GitLab on all sites. WARNING: Geo works with streaming replication. Logical replication is not supported at this time. There is an [issue where support is being discussed](https://gitlab.com/gitlab-org/gitlab/-/issues/7420). -#### Step 1. Configure the **primary** server +#### Step 1. Configure the **primary** site -1. SSH into your GitLab **primary** server and login as root: +1. SSH into your GitLab **primary** site and login as root: ```shell sudo -i @@ -81,13 +81,13 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o gitlab_rails['geo_node_name'] = '<site_name_here>' ``` -1. Reconfigure the **primary** node for the change to take effect: +1. Reconfigure the **primary** site for the change to take effect: ```shell gitlab-ctl reconfigure ``` -1. Execute the command below to define the node as **primary** node: +1. Execute the command below to define the site as **primary** site: ```shell gitlab-ctl set-geo-primary-node @@ -154,17 +154,17 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o 1. Configure PostgreSQL to listen on network interfaces: For security reasons, PostgreSQL does not listen on any network interfaces - by default. However, Geo requires the **secondary** node to be able to - connect to the **primary** node's database. For this reason, we need the address of - each node. + by default. However, Geo requires the **secondary** site to be able to + connect to the **primary** site's database. For this reason, we need the IP address of + each site. NOTE: For external PostgreSQL instances, see [additional instructions](external_database.md). If you are using a cloud provider, you can lookup the addresses for each - Geo node through your cloud provider's management console. + Geo site through your cloud provider's management console. - To lookup the address of a Geo node, SSH in to the Geo node and execute: + To lookup the address of a Geo site, SSH in to the Geo site and execute: ```shell ## @@ -183,11 +183,11 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o | Configuration | Address | |:----------------------------------------|:----------------------------------------------------------------------| - | `postgresql['listen_address']` | **Primary** node's public or VPC private address. | - | `postgresql['md5_auth_cidr_addresses']` | **Primary** and **Secondary** nodes' public or VPC private addresses. | + | `postgresql['listen_address']` | **Primary** site's public or VPC private address. | + | `postgresql['md5_auth_cidr_addresses']` | **Primary** and **Secondary** sites' public or VPC private addresses. | If you are using Google Cloud Platform, SoftLayer, or any other vendor that - provides a virtual private cloud (VPC) you can use the **primary** and **secondary** nodes + provides a virtual private cloud (VPC) you can use the **primary** and **secondary** sites private addresses (corresponds to "internal address" for Google Cloud Platform) for `postgresql['md5_auth_cidr_addresses']` and `postgresql['listen_address']`. @@ -201,10 +201,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o `127.0.0.1`. For more information, see [omnibus-5258](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5258). Depending on your network configuration, the suggested addresses may not - be correct. If your **primary** node and **secondary** nodes connect over a local + be correct. If your **primary** site and **secondary** sites connect over a local area network, or a virtual network connecting availability zones like [Amazon's VPC](https://aws.amazon.com/vpc/) or [Google's VPC](https://cloud.google.com/vpc/) - you should use the **secondary** node's private address for `postgresql['md5_auth_cidr_addresses']`. + you should use the **secondary** site's private address for `postgresql['md5_auth_cidr_addresses']`. Edit `/etc/gitlab/gitlab.rb` and add the following, replacing the IP addresses with addresses appropriate to your network configuration: @@ -225,19 +225,18 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## Primary address ## - replace '<primary_node_ip>' with the public or VPC address of your Geo primary node ## - postgresql['listen_address'] = '<primary_node_ip>' + postgresql['listen_address'] = '<primary_site_ip>' ## # Allow PostgreSQL client authentication from the primary and secondary IPs. These IPs may be # public or VPC addresses in CIDR format, for example ['198.51.100.1/32', '198.51.100.2/32'] ## - postgresql['md5_auth_cidr_addresses'] = ['<primary_node_ip>/32', '<secondary_node_ip>/32'] + postgresql['md5_auth_cidr_addresses'] = ['<primary_site_ip>/32', '<secondary_site_ip>/32'] ## ## Replication settings - ## - set this to be the number of Geo secondary nodes you have ## - postgresql['max_replication_slots'] = 1 + # postgresql['max_replication_slots'] = 1 # Set this to be the number of Geo secondary nodes if you have more than one # postgresql['max_wal_senders'] = 10 # postgresql['wal_keep_segments'] = 10 @@ -248,10 +247,10 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o gitlab_rails['auto_migrate'] = false ``` -1. Optional: If you want to add another **secondary** node, the relevant setting would look like: +1. Optional: If you want to add another **secondary** site, the relevant setting would look like: ```ruby - postgresql['md5_auth_cidr_addresses'] = ['<primary_node_ip>/32', '<secondary_node_ip>/32', '<another_secondary_node_ip>/32'] + postgresql['md5_auth_cidr_addresses'] = ['<primary_site_ip>/32', '<secondary_site_ip>/32', '<another_secondary_site_ip>/32'] ``` You may also want to edit the `wal_keep_segments` and `max_wal_senders` to match your @@ -288,20 +287,20 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o 1. Now that the PostgreSQL server is set up to accept remote connections, run `netstat -plnt | grep 5432` to make sure that PostgreSQL is listening on port - `5432` to the **primary** server's private address. + `5432` to the **primary** site's private address. 1. A certificate was automatically generated when GitLab was reconfigured. This is used automatically to protect your PostgreSQL traffic from eavesdroppers, but to protect against active ("man-in-the-middle") attackers, - the **secondary** node needs a copy of the certificate. Make a copy of the PostgreSQL - `server.crt` file on the **primary** node by running this command: + the **secondary** site needs a copy of the certificate. Make a copy of the PostgreSQL + `server.crt` file on the **primary** site by running this command: ```shell cat ~gitlab-psql/data/server.crt ``` Copy the output into a clipboard or into a local file. You - need it when setting up the **secondary** node! The certificate is not sensitive + need it when setting up the **secondary** site! The certificate is not sensitive data. However, this certificate is created with a generic `PostgreSQL` Common Name. For this, @@ -324,7 +323,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o #### Step 2. Configure the **secondary** server -1. SSH into your GitLab **secondary** server and login as root: +1. SSH into your GitLab **secondary** site and login as root: ```shell sudo -i @@ -338,28 +337,28 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` NOTE: - This step is important so we don't try to execute anything before the node is fully configured. + This step is important so we don't try to execute anything before the site is fully configured. -1. [Check TCP connectivity](../../raketasks/maintenance.md) to the **primary** node's PostgreSQL server: +1. [Check TCP connectivity](../../raketasks/maintenance.md) to the **primary** site's PostgreSQL server: ```shell - gitlab-rake gitlab:tcp_check[<primary_node_ip>,5432] + gitlab-rake gitlab:tcp_check[<primary_site_ip>,5432] ``` NOTE: If this step fails, you may be using the wrong IP address, or a firewall may - be preventing access to the server. Check the IP address, paying close + be preventing access to the site. Check the IP address, paying close attention to the difference between public and private addresses and ensure - that, if a firewall is present, the **secondary** node is permitted to connect to the - **primary** node on port 5432. + that, if a firewall is present, the **secondary** site is permitted to connect to the + **primary** site on port 5432. -1. Create a file `server.crt` in the **secondary** server, with the content you got on the last step of the **primary** node's setup: +1. Create a file `server.crt` in the **secondary** site, with the content you got on the last step of the **primary** site's setup: ```shell editor server.crt ``` -1. Set up PostgreSQL TLS verification on the **secondary** node: +1. Set up PostgreSQL TLS verification on the **secondary** site: Install the `server.crt` file: @@ -374,9 +373,9 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o PostgreSQL now only recognizes that exact certificate when verifying TLS connections. The certificate can only be replicated by someone with access - to the private key, which is **only** present on the **primary** node. + to the private key, which is **only** present on the **primary** site. -1. Test that the `gitlab-psql` user can connect to the **primary** node's database +1. Test that the `gitlab-psql` user can connect to the **primary** site's database (the default Omnibus database name is `gitlabhq_production`): ```shell @@ -386,7 +385,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o -U gitlab_replicator \ -d "dbname=gitlabhq_production sslmode=verify-ca" \ -W \ - -h <primary_node_ip> + -h <primary_site_ip> ``` NOTE: @@ -397,11 +396,11 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o When prompted enter the _plaintext_ password you set in the first step for the `gitlab_replicator` user. If all worked correctly, you should see - the list of **primary** node's databases. + the list of **primary** site's databases. A failure to connect here indicates that the TLS configuration is incorrect. - Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** node - match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** node. + Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** site + match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** site. 1. Configure PostgreSQL: @@ -420,14 +419,14 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ## ## Secondary address - ## - replace '<secondary_node_ip>' with the public or VPC address of your Geo secondary node + ## - replace '<secondary_site_ip>' with the public or VPC address of your Geo secondary site ## - postgresql['listen_address'] = '<secondary_node_ip>' - postgresql['md5_auth_cidr_addresses'] = ['<secondary_node_ip>/32'] + postgresql['listen_address'] = '<secondary_site_ip>' + postgresql['md5_auth_cidr_addresses'] = ['<secondary_site_ip>/32'] ## - ## Database credentials password (defined previously in primary node) - ## - replicate same values here as defined in primary node + ## Database credentials password (defined previously in primary site) + ## - replicate same values here as defined in primary site ## postgresql['sql_replication_password'] = '<md5_hash_of_your_password>' postgresql['sql_user_password'] = '<md5_hash_of_your_password>' @@ -435,7 +434,7 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ``` For external PostgreSQL instances, see [additional instructions](external_database.md). - If you bring a former **primary** node back online to serve as a **secondary** node, then you also must remove `roles(['geo_primary_role'])` or `geo_primary_role['enable'] = true`. + If you bring a former **primary** site back online to serve as a **secondary** site, then you also must remove `roles(['geo_primary_role'])` or `geo_primary_role['enable'] = true`. 1. Reconfigure GitLab for the changes to take effect: @@ -451,24 +450,24 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o #### Step 3. Initiate the replication process -Below we provide a script that connects the database on the **secondary** node to -the database on the **primary** node, replicates the database, and creates the +Below we provide a script that connects the database on the **secondary** site to +the database on the **primary** site, replicates the database, and creates the needed files for streaming replication. The directories used are the defaults that are set up in Omnibus. If you have changed any defaults, configure it as you see fit replacing the directories and paths. WARNING: -Make sure to run this on the **secondary** server as it removes all PostgreSQL's +Make sure to run this on the **secondary** site as it removes all PostgreSQL's data before running `pg_basebackup`. -1. SSH into your GitLab **secondary** server and login as root: +1. SSH into your GitLab **secondary** site and login as root: ```shell sudo -i ``` -1. Choose a database-friendly name to use for your **secondary** node to +1. Choose a database-friendly name to use for your **secondary** site to use as the replication slot name. For example, if your domain is `secondary.geo.example.com`, you may use `secondary_example` as the slot name as shown in the commands below. @@ -476,13 +475,13 @@ data before running `pg_basebackup`. 1. Execute the command below to start a backup/restore and begin the replication WARNING: - Each Geo **secondary** node must have its own unique replication slot name. + Each Geo **secondary** site must have its own unique replication slot name. Using the same slot name between two secondaries breaks PostgreSQL replication. ```shell gitlab-ctl replicate-geo-database \ - --slot-name=<secondary_node_name> \ - --host=<primary_node_ip> \ + --slot-name=<secondary_site_name> \ + --host=<primary_site_ip> \ --sslmode=verify-ca ``` @@ -516,7 +515,7 @@ data before running `pg_basebackup`. - Change the `--slot-name` to the name of the replication slot to be used on the **primary** database. The script attempts to create the replication slot automatically if it does not exist. - - If you're repurposing an old server into a Geo **secondary** node, you must + - If you're repurposing an old site into a Geo **secondary** site, you must add `--force` to the command line. - When not in a production machine you can disable backup step if you really sure this is what you want by adding `--skip-backup` @@ -540,7 +539,7 @@ see [High Availability with Omnibus GitLab](../../postgresql/replication_and_fai To change the password for the [replication user](https://wiki.postgresql.org/wiki/Streaming_Replication) when using Omnibus-managed PostgreSQL instances: -On the GitLab Geo **primary** server: +On the GitLab Geo **primary** site: 1. The default value for the replication user is `gitlab_replicator`, but if you've set a custom replication user in your `/etc/gitlab/gitlab.rb` under the `postgresql['sql_replication_user']` setting, make sure to @@ -574,19 +573,18 @@ On the GitLab Geo **primary** server: sudo gitlab-ctl restart postgresql ``` -Until the password is updated on any **secondary** servers, the [PostgreSQL log](../../logs.md#postgresql-logs) on +Until the password is updated on any **secondary** sites, the [PostgreSQL log](../../logs/index.md#postgresql-logs) on the secondaries will report the following error message: ```console FATAL: could not connect to the primary server: FATAL: password authentication failed for user "gitlab_replicator" ``` -On all GitLab Geo **secondary** servers: +On all GitLab Geo **secondary** sites: 1. The first step isn't necessary from a configuration perspective, because the hashed `'sql_replication_password'` - is not used on the GitLab Geo **secondary**. However in the event that **secondary** needs to be promoted - to the GitLab Geo **primary**, make sure to match the `'sql_replication_password'` in the secondary - server configuration. + is not used on the GitLab Geo **secondary** sites. However in the event that **secondary** site needs to be promoted + to the GitLab Geo **primary**, make sure to match the `'sql_replication_password'` in the **secondary** site configuration. Edit `/etc/gitlab/gitlab.rb`: @@ -616,6 +614,28 @@ In GitLab 14.0, Patroni replaced `repmgr` as the supported NOTE: If you still haven't [migrated from repmgr to Patroni](#migrating-from-repmgr-to-patroni) you're highly advised to do so. +### Migrating from repmgr to Patroni + +1. Before migrating, we recommend that there is no replication lag between the **primary** and **secondary** sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. +1. Follow the [instructions to migrate repmgr to Patroni](../../postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni). When configuring Patroni on each **primary** site database node, add `patroni['replication_slots'] = { '<slot_name>' => 'physical' }` +to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your **secondary** site. This ensures that Patroni recognizes the replication slot as permanent and not drop it upon restarting. +1. If database replication to the **secondary** site was paused before migration, resume replication once Patroni is confirmed working on the **primary** site. + +### Migrating a single PostgreSQL node to Patroni + +Before the introduction of Patroni, Geo had no Omnibus support for HA setups on the **secondary** site. + +With Patroni it's now possible to support that. To migrate the existing PostgreSQL to Patroni: + +1. Make sure you have a Consul cluster setup on the secondary (similar to how you set it up on the **primary** site). +1. [Configure a permanent replication slot](#step-1-configure-patroni-permanent-replication-slot-on-the-primary-site). +1. [Configure the internal load balancer](#step-2-configure-the-internal-load-balancer-on-the-primary-site). +1. [Configure a PgBouncer node](#step-3-configure-pgbouncer-nodes-on-the-secondary-site) +1. [Configure a Standby Cluster](#step-4-configure-a-standby-cluster-on-the-secondary-site) + on that single node machine. + +You end up with a “Standby Cluster” with a single node. That allows you to later on add additional Patroni nodes by following the same instructions above. + ### Patroni support Patroni is the official replication management solution for Geo. It @@ -648,11 +668,11 @@ and other database best practices. ##### Step 1. Configure Patroni permanent replication slot on the primary site -To set up database replication with Patroni on a secondary node, we must -configure a _permanent replication slot_ on the primary node's Patroni cluster, +To set up database replication with Patroni on a secondary site, we must +configure a _permanent replication slot_ on the primary site's Patroni cluster, and ensure password authentication is used. -For each Patroni instance on the primary site **starting on the Patroni +On each node running a Patroni instance on the primary site **starting on the Patroni Leader instance**: 1. SSH into your Patroni instance and login as root: @@ -755,7 +775,7 @@ backend postgresql Refer to your preferred Load Balancer's documentation for further guidance. -##### Step 3. Configure a PgBouncer node on the secondary site +##### Step 3. Configure PgBouncer nodes on the secondary site A production-ready and highly available configuration requires at least three Consul nodes, a minimum of one PgBouncer node, but it's recommended to have @@ -764,7 +784,7 @@ more than one PgBouncer service nodes. The internal load balancer provides a sin endpoint for connecting to the PgBouncer cluster. For more information, see [High Availability with Omnibus GitLab](../../postgresql/replication_and_failover.md). -Follow the minimal configuration for the PgBouncer node: +On each node running a PgBouncer instance on the **secondary** site: 1. SSH into your PgBouncer node and login as root: @@ -820,11 +840,10 @@ Follow the minimal configuration for the PgBouncer node: ##### Step 4. Configure a Standby cluster on the secondary site NOTE: -If you are converting a secondary site to a Patroni Cluster, you must start -on the PostgreSQL instance. It becomes the Patroni Standby Leader instance, -and then you can switchover to another replica if you need. +If you are converting a secondary site with a single PostgreSQL instance to a Patroni Cluster, you must start on the PostgreSQL instance. It becomes the Patroni Standby Leader instance, +and then you can switch over to another replica if you need. -For each Patroni instance on the secondary site: +For each node running a Patroni instance on the secondary site: 1. SSH into your Patroni node and login as root: @@ -894,39 +913,29 @@ For each Patroni instance on the secondary site: gitlab-ctl start patroni ``` -### Migrating from repmgr to Patroni - -1. Before migrating, we recommend that there is no replication lag between the primary and secondary sites and that replication is paused. In GitLab 13.2 and later, you can pause and resume replication with `gitlab-ctl geo-replication-pause` and `gitlab-ctl geo-replication-resume` on a Geo secondary database node. -1. Follow the [instructions to migrate repmgr to Patroni](../../postgresql/replication_and_failover.md#switching-from-repmgr-to-patroni). When configuring Patroni on each primary site database node, add `patroni['replication_slots'] = { '<slot_name>' => 'physical' }` -to `gitlab.rb` where `<slot_name>` is the name of the replication slot for your Geo secondary. This ensures that Patroni recognizes the replication slot as permanent and not drop it upon restarting. -1. If database replication to the secondary was paused before migration, resume replication once Patroni is confirmed working on the primary. - -### Migrating a single PostgreSQL node to Patroni - -Before the introduction of Patroni, Geo had no Omnibus support for HA setups on the secondary node. +### Migrating a single tracking database node to Patroni -With Patroni it's now possible to support that. To migrate the existing PostgreSQL to Patroni: +Before the introduction of Patroni, Geo had no Omnibus support for HA setups on +the secondary site. -1. Make sure you have a Consul cluster setup on the secondary (similar to how you set it up on the primary). -1. [Configure a permanent replication slot](#step-1-configure-patroni-permanent-replication-slot-on-the-primary-site). -1. [Configure the internal load balancer](#step-2-configure-the-internal-load-balancer-on-the-primary-site). -1. [Configure a PgBouncer node](#step-3-configure-a-pgbouncer-node-on-the-secondary-site) -1. [Configure a Standby Cluster](#step-4-configure-a-standby-cluster-on-the-secondary-site) - on that single node machine. +With Patroni, it's now possible to support that. Due to some restrictions on the +Patroni implementation on Omnibus that do not allow us to manage two different +clusters on the same machine, we recommend setting up a new Patroni cluster for +the tracking database by following the same instructions above. -You end up with a "Standby Cluster" with a single node. That allows you to later on add additional Patroni nodes -by following the same instructions above. +The secondary nodes backfill the new tracking database, and no data +synchronization is required. ### Configuring Patroni cluster for the tracking PostgreSQL database -Secondary sites use a separate PostgreSQL installation as a tracking database to +**Secondary** sites use a separate PostgreSQL installation as a tracking database to keep track of replication status and automatically recover from potential replication issues. Omnibus automatically configures a tracking database when `roles(['geo_secondary_role'])` is set. If you want to run this database in a highly available configuration, don't use the `geo_secondary_role` above. Instead, follow the instructions below. -A production-ready and secure setup requires at least three Consul nodes, two +A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes, two Patroni nodes and one PgBouncer node on the secondary site. Because of [omnibus-6587](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6587), Consul can't track multiple @@ -935,9 +944,9 @@ services, so these must be different than the nodes used for the Standby Cluster Be sure to use [password credentials](../../postgresql/replication_and_failover.md#database-authorization-for-patroni) and other database best practices. -#### Step 1. Configure a PgBouncer node on the secondary site +#### Step 1. Configure PgBouncer nodes on the secondary site -Follow the minimal configuration for the PgBouncer node for the tracking database: +On each node running the PgBouncer service for the PostgreSQL tracking database: 1. SSH into your PgBouncer node and login as root: @@ -999,7 +1008,7 @@ Follow the minimal configuration for the PgBouncer node for the tracking databas #### Step 2. Configure a Patroni cluster -For each Patroni instance on the secondary site for the tracking database: +On each node running a Patroni instance on the secondary site for the PostgreSQL tracking database: 1. SSH into your Patroni node and login as root: @@ -1063,7 +1072,7 @@ For each Patroni instance on the secondary site for the tracking database: gitlab-ctl reconfigure ``` -#### Step 3. Configure the tracking database on the secondary nodes +#### Step 3. Configure the tracking database on the secondary sites For each node running the `gitlab-rails`, `sidekiq`, and `geo-logcursor` services: @@ -1100,19 +1109,6 @@ For each node running the `gitlab-rails`, `sidekiq`, and `geo-logcursor` service gitlab-rake db:migrate:geo ``` -### Migrating a single tracking database node to Patroni - -Before the introduction of Patroni, Geo had no Omnibus support for HA setups on -the secondary node. - -With Patroni, it's now possible to support that. Due to some restrictions on the -Patroni implementation on Omnibus that do not allow us to manage two different -clusters on the same machine, we recommend setting up a new Patroni cluster for -the tracking database by following the same instructions above. - -The secondary nodes backfill the new tracking database, and no data -synchronization is required. - ## Troubleshooting Read the [troubleshooting document](../replication/troubleshooting.md). diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md index 5ddfee6774e..79b52ef71da 100644 --- a/doc/administration/geo/setup/index.md +++ b/doc/administration/geo/setup/index.md @@ -22,8 +22,10 @@ The steps below should be followed in the order they appear. **Make sure the Git If you installed GitLab using the Omnibus packages (highly recommended): +1. Confirm the [requirements for running Geo](../index.md#requirements-for-running-geo) are met. 1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the nodes that serve as the **secondary** site. **Do not create an account or log in** to the new **secondary** site. The **GitLab version must match** across primary and secondary sites. 1. [Add the GitLab License](../../../user/admin_area/license.md) on the **primary** site to unlock Geo. The license must be for [GitLab Premium](https://about.gitlab.com/pricing/) or higher. +1. [Confirm network connectivity](../index.md#firewall-rules) between the **primary** and **secondary** site. 1. [Set up the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology). 1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** sites. 1. [Configure GitLab](../replication/configuration.md) to set the **primary** and **secondary** sites. diff --git a/doc/administration/get_started.md b/doc/administration/get_started.md index b2bdd876499..44b09ef185a 100644 --- a/doc/administration/get_started.md +++ b/doc/administration/get_started.md @@ -37,8 +37,8 @@ Watch an overview of [groups and projects](https://www.youtube.com/watch?v=cqb2m Get started: - Create a [project](../user/project/working_with_projects.md#create-a-project). -- Create a [group](../user/group/index.md#create-a-group). -- [Add members](../user/group/index.md#add-users-to-a-group) to the group. +- Create a [group](../user/group/manage.md#create-a-group). +- [Add members](../user/group/manage.md#add-users-to-a-group) to the group. - Create a [subgroup](../user/group/subgroups/index.md#create-a-subgroup). - [Add members](../user/group/subgroups/index.md#subgroup-membership) to the subgroup. - Enable [external authorization control](../user/admin_area/settings/external_authorization.md#configuration). @@ -144,7 +144,7 @@ You can restore a backup only to **the exact same version and type** (Community ### Back up GitLab SaaS -Backups of GitLab databases and filesystems are taken every 24 hours, and are kept for two weeks on a rolling schedule. All backups are encrypted. +Backups of GitLab databases and file systems are taken every 24 hours, and are kept for two weeks on a rolling schedule. All backups are encrypted. - GitLab SaaS creates backups to ensure your data is secure, but you can't use these methods to export or back up your data yourself. - Issues are stored in the database. They can't be stored in Git itself. diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 4b2832bebc0..5b868c274cd 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -280,8 +280,9 @@ Updates to example must be made at: ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml` - to confirm that Gitaly can perform callbacks to the GitLab internal API. +1. Confirm that Gitaly can perform callbacks to the GitLab internal API: + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. **For installations from source** @@ -330,8 +331,9 @@ Updates to example must be made at: ``` 1. Save the files and [restart GitLab](../restart_gitlab.md#installations-from-source). -1. Run `sudo -u git /home/git/gitaly/gitaly-hooks check /home/git/gitaly/config.toml` - to confirm that Gitaly can perform callbacks to the GitLab internal API. +1. Confirm that Gitaly can perform callbacks to the GitLab internal API: + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. WARNING: If directly copying repository data from a GitLab server to Gitaly, ensure that the metadata file, @@ -792,7 +794,7 @@ gitaly['concurrency'] = [ - `max_per_repo` is the maximum number of in-flight RPC calls for the given RPC per repository. - `max_queue_time` is the maximum amount of time a request can wait in the concurrency queue to be picked up by Gitaly. -- `max_queue_size` is the maximum size the concurrency queue can grow to before requests are rejected by +- `max_queue_size` is the maximum size the concurrency queue (per RPC method) can grow to before requests are rejected by Gitaly. This limits the number of in-flight RPC calls for the given RPCs. The limit is applied per @@ -811,26 +813,12 @@ information, see the [relevant documentation](monitoring.md#monitor-gitaly-concu ## Control groups FLAG: -On self-managed GitLab, by default cgroups are not available. To make it available, ask an administrator to +On self-managed GitLab, by default repository cgroups are not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) named `gitaly_run_cmds_in_cgroup`. -Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations, -especially for large repositories. - Control groups (cgroups) in Linux allow limits to be imposed on how much memory and CPU can be consumed. See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information. -cgroups can be useful for protecting the system against resource exhaustion because of overcomsumption of memory and CPU. - -Gitaly has built-in cgroups control. When configured, Gitaly assigns Git -processes to a cgroup based on the repository the Git command is operating in. -Each cgroup has a memory and CPU limit. When a cgroup reaches its: - -- Memory limit, the kernel looks through the processes for a candidate to kill. -- CPU limit, processes are not killed, but the processes are prevented from consuming more CPU than allowed. - -The main reason to configure cgroups for your GitLab installation is that it -protects against system resource starvation due to a few large repositories or -bad actors. +cgroups can be useful for protecting the system against resource exhaustion because of over consumption of memory and CPU. Some Git operations are expensive by nature. `git clone`, for instance, spawns a `git-upload-pack` process on the server that can consume a lot of memory @@ -838,33 +826,33 @@ for large repositories. For example, a client that keeps on cloning a large repository over and over again. This situation could potentially use up all of the memory on a server, causing other operations to fail for other users. -There are many ways someone can create a repository that can consume large amounts of memory when cloned or downloaded. +A repository can consume large amounts of memory for many reasons when cloned or downloaded. Using cgroups allows the kernel to kill these operations before they hog up all system resources. -### Configure cgroups in Gitaly +Gitaly shells out to Git for many of its operations. Git can consume a lot of resources for certain operations, +especially for large repositories. -Two ways of configuring cgroups are available. +Gitaly has built-in cgroups control. When configured, Gitaly assigns Git processes to a cgroup based on the repository +the Git command is operating in. These cgroups are called repository cgroups. Each repository cgroup: -#### Configure cgroups (new method) +- Has a memory and CPU limit. +- Contains the Git processes for a single repository. +- Uses a consistent hash to ensure a Git process for a given repository always ends up in the same cgroup. -> This method of configuring cgroups introduced in GitLab 15.1. +When a repository cgroup reaches its: -Gitaly creates a pool of cgroups that are isolated based on the repository used in the Git command to be placed under one of these cgroups. +- Memory limit, the kernel looks through the processes for a candidate to kill. +- CPU limit, processes are not killed, but the processes are prevented from consuming more CPU than allowed. -To configure cgroups in Gitaly, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. +You configure repository cgroups for your GitLab installation to protect against system resource starvation from a few +large repositories or bad actors. -For example: +### Configure repository cgroups (new method) -```ruby -# in /etc/gitlab/gitlab.rb -gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" -gitaly['cgroups_hierarchy_root'] =>"gitaly" -gitaly['cgroups_memory_bytes'] = 64424509440, # 60gb -gitaly['cgroups_cpu_shares'] = 1024 -gitaly['cgroups_repositories_count'] => 1000, -gitaly['cgroups_repositories_memory_bytes'] => 32212254720 # 20gb -gitaly['cgroups_repositories_cpu_shares'] => 512 -``` +> This method of configuring repository cgroups was introduced in GitLab 15.1. + +To configure repository cgroups in Gitaly using the new method, use the following settings for the new configuration method +to `gitaly['cgroups']` in `/etc/gitlab/gitlab.rb`: - `cgroups_mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. - `cgroups_hierarchy_root` is the parent cgroup under which Gitaly creates groups, and @@ -873,7 +861,7 @@ gitaly['cgroups_repositories_cpu_shares'] => 512 when Gitaly starts. - `cgroups_memory_bytes` is the total memory limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. -- `cgroups_cpu_shares` is the cpu limit that is imposed collectively on all Git +- `cgroups_cpu_shares` is the CPU limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. - `cgroups_repositories_count` is the number of cgroups in the cgroups pool. Each time a new Git @@ -881,30 +869,30 @@ gitaly['cgroups_repositories_cpu_shares'] => 512 on the repository the command is for. A circular hashing algorithm assigns Git commands to these cgroups, so a Git command for a repository is always assigned to the same cgroup. -- `cgroups_repositories_memory_bytes` is the total memory limit that is imposed collectively on all - Git processes that Gitaly spawns. 0 implies no limit. This value cannot exceed - that of the top level `cgroups_memory_bytes`. -- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed collectively on all Git - processes Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, - which represents 100% of CPU. This value cannot exceed that of the top - level`cgroups_cpu_shares`. - -#### Configure cgroups (legacy method) +- `cgroups_repositories_memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup. + 0 implies no limit. This value cannot exceed that of the top level `cgroups_memory_bytes`. +- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup. + 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. + This value cannot exceed that of the top level`cgroups_cpu_shares`. -To configure cgroups in Gitaly for GitLab versions using the legacy method, add `gitaly['cgroups']` to `/etc/gitlab/gitlab.rb`. For -example: +For example: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['cgroups_count'] = 1000 gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" -gitaly['cgroups_hierarchy_root'] = "gitaly" -gitaly['cgroups_memory_limit'] = 32212254720 -gitaly['cgroups_memory_enabled'] = true +gitaly['cgroups_hierarchy_root'] => "gitaly" +gitaly['cgroups_memory_bytes'] = 64424509440, # 60gb gitaly['cgroups_cpu_shares'] = 1024 -gitaly['cgroups_cpu_enabled'] = true +gitaly['cgroups_repositories_count'] => 1000, +gitaly['cgroups_repositories_memory_bytes'] => 32212254720 # 20gb +gitaly['cgroups_repositories_cpu_shares'] => 512 ``` +### Configure repository cgroups (legacy method) + +To configure repository cgroups in Gitaly using the legacy method, use the following settings +in `/etc/gitlab/gitlab.rb`: + - `cgroups_count` is the number of cgroups created. Each time a new command is spawned, Gitaly assigns it to one of these cgroups based on the command line arguments of the command. A circular hashing algorithm assigns @@ -917,7 +905,21 @@ gitaly['cgroups_cpu_enabled'] = true - `cgroups_memory_enabled` enables or disables the memory limit on cgroups. - `cgroups_memory_bytes` is the total memory limit each cgroup imposes on the processes added to it. - `cgroups_cpu_enabled` enables or disables the CPU limit on cgroups. -- `cgroups_cpu_shares` is the CPU limit each cgroup imposes on the processes added to it. The maximum is 1024 shares, which represents 100% of CPU. +- `cgroups_cpu_shares` is the CPU limit each cgroup imposes on the processes added to it. The maximum is 1024 shares, + which represents 100% of CPU. + +For example: + +```ruby +# in /etc/gitlab/gitlab.rb +gitaly['cgroups_count'] = 1000 +gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" +gitaly['cgroups_hierarchy_root'] = "gitaly" +gitaly['cgroups_memory_limit'] = 32212254720 +gitaly['cgroups_memory_enabled'] = true +gitaly['cgroups_cpu_shares'] = 1024 +gitaly['cgroups_cpu_enabled'] = true +``` ### Configuring oversubscription @@ -926,16 +928,15 @@ In the previous example using the new configuration method: - The top level memory limit is capped at 60gb. - Each of the 1000 cgroups in the repositories pool is capped at 20gb. -This is called "oversubscription". Each cgroup in the pool has a much larger capacity than 1/1000th +This configuration leads to "oversubscription". Each cgroup in the pool has a much larger capacity than 1/1000th of the top-level memory limit. This strategy has two main benefits: -- It gives the host protection from overall memory starvation (OOM), because the top-level - cgroup's memory limit can be set to a threshold smaller than the host's - capacity. Processes outside of that cgroup are not at risk of OOM. +- It gives the host protection from overall memory starvation (OOM), because the memory limit of the top-level cgroup + can be set to a threshold smaller than the host's capacity. Processes outside of that cgroup are not at risk of OOM. - It allows each individual cgroup in the pool to burst up to a generous upper - bound (in this example 20 GB) that is smaller than the parent cgroup's limit, + bound (in this example 20 GB) that is smaller than the limit of the parent cgroup, but substantially larger than 1/N of the parent's limit. In this example, up to 3 child cgroups can concurrently burst up to their max. In general, all 1000 cgroups would use much less than the 20 GB. @@ -1141,8 +1142,7 @@ gitaly['pack_objects_cache_enabled'] = true #### `enabled` defaults to `false` The cache is disabled by default. This is because in some cases, it -can create an [extreme -increase](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4010#note_534564684) +can create an [extreme increase](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4010#note_534564684) in the number of bytes written to disk. On GitLab.com, we have verified that our repository storage disks can handle this extra workload, but we felt we cannot assume this is true everywhere. @@ -1302,14 +1302,26 @@ process repositories that do not pass consistency checks. For Omnibus GitLab installations, edit `/etc/gitlab/gitlab.rb` and set the following keys (in this example, to disable the `hasDotgit` consistency check): -```ruby -ignored_git_errors = ["hasDotgit = ignore"] -omnibus_gitconfig['system'] = { - "fsck" => ignored_git_errors, - "fetch.fsck" => ignored_git_errors, - "receive.fsck" => ignored_git_errors, -} -``` +- In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later: + + ```ruby + gitaly['gitconfig'] = [ + { key: "fsck.hasDotgit", value: "ignore" }, + { key: "fetch.fsck.hasDotgit", value: "ignore" }, + { key: "receive.fsck.hasDotgit", value: "ignore "}, + ] + ``` + +- In GitLab 15.2 and earlier (legacy method): + + ```ruby + ignored_git_errors = ["hasDotgit = ignore"] + omnibus_gitconfig['system'] = { + "fsck" => ignored_git_errors, + "fetch.fsck" => ignored_git_errors, + "receive.fsck" => ignored_git_errors, + } + ``` For source installs, edit the Gitaly configuration (`gitaly.toml`) to do the equivalent: diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index c543f62f135..b053da7ac9b 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -70,7 +70,7 @@ the current status of these issues, please refer to the referenced issues and ep | Issue | Summary | How to avoid | |:--------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------| -| Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. Work is in-progress to update Gitaly Cluster to [identify repositories with a unique and persistent identifier](https://gitlab.com/gitlab-org/gitaly/-/issues/3485), which is expected to resolve the issue. | No known solution at this time. | +| Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. | No known solution prior to GitLab 15.0. In GitLab 15.0 to 15.2, enable the [`gitaly_praefect_generated_replica_paths` feature flag](#praefect-generated-replica-paths-gitlab-150-and-later). In GitLab 15.3, the feature flag is enabled by default. | | Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform normal operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting upgrade assistance](https://about.gitlab.com/support/scheduling-upgrade-assistance/) so your upgrade plan can be reviewed by support. | | Restoring a Gitaly Cluster node from a snapshot in a running cluster | Because the Gitaly Cluster runs with consistent state, introducing a single node that is behind will result in the cluster not being able to reconcile the nodes data and other nodes data | Don't restore a single Gitaly Cluster node from a backup snapshot. If you must restore from backup, it's best to snapshot all Gitaly Cluster nodes at the same time and take a database dump of the Praefect database. | @@ -80,16 +80,31 @@ Gitaly Cluster does not support snapshot backups. Snapshot backups can cause iss out of sync with the disk storage. Because of how Praefect rebuilds the replication metadata of Gitaly disk information during a restore, we recommend using the [official backup and restore Rake tasks](../../raketasks/backup_restore.md). -If you are unable to use this method, please contact customer support for restoration help. +The [incremental backup method](../../raketasks/backup_gitlab.md#incremental-repository-backups) +can be used to speed up Gitaly Cluster backups. -We are tracking in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351383) improvements to the -[official backup and restore Rake tasks](../../raketasks/backup_restore.md) to add support for incremental backups. For -more information, see [this epic](https://gitlab.com/groups/gitlab-org/-/epics/2094). +If you are unable to use either method, please contact customer support for restoration help. ### What to do if you are on Gitaly Cluster experiencing an issue or limitation Please contact customer support for immediate help in restoration or recovery. +## Directly accessing repositories + +GitLab doesn't advise directly accessing Gitaly repositories stored on disk with a Git client or any other tool, +because Gitaly is being continuously improved and changed. These improvements may invalidate +your assumptions, resulting in performance degradation, instability, and even data loss. For example: + +- Gitaly has optimizations such as the [`info/refs` advertisement cache](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/design_diskcache.md), + that rely on Gitaly controlling and monitoring access to repositories by using the official gRPC + interface. +- [Gitaly Cluster](#gitaly-cluster) has optimizations, such as fault tolerance and + [distributed reads](#distributed-reads), that depend on the gRPC interface and database + to determine repository state. + +WARNING: +Accessing Git repositories directly is done at your own risk and is not supported. + ## Gitaly The following shows GitLab set up to use direct access to Gitaly: @@ -141,6 +156,11 @@ Gitaly comes pre-configured with Omnibus GitLab, which is a configuration GitLab installations for more than 2000 active users performing daily Git write operation may be best suited by using Gitaly Cluster. +### Backing up repositories + +When backing up or syncing repositories using tools other than GitLab, you must [prevent writes](../../raketasks/backup_restore.md#prevent-writes-and-copy-the-git-repository-data) +while copying repository data. + ## Gitaly Cluster Git storage is provided through the Gitaly service in GitLab, and is essential to the operation of @@ -297,11 +317,12 @@ follow the [hashed storage](../repository_storage_types.md#hashed-storage) schem #### Praefect-generated replica paths (GitLab 15.0 and later) -> Introduced in GitLab 15.0 behind [a feature flag](https://gitlab.com/gitlab-org/gitaly/-/issues/4218) named `gitaly_praefect_generated_replica_paths`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4218) in GitLab 15.0 [with a flag](../feature_flags.md) named `gitaly_praefect_generated_replica_paths`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitaly/-/issues/4218) in GitLab 15.2. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4809) in GitLab 15.3. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../feature_flags.md) -named `gitaly_praefect_generated_replica_paths`. On GitLab.com, this feature is available but can be configured by GitLab.com administrators only. The feature is not ready for production use. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../feature_flags.md) named `gitaly_praefect_generated_replica_paths`. On GitLab.com, this feature is available but can be configured by GitLab.com administrators only. When Gitaly Cluster creates a repository, it assigns the repository a unique and permanent ID called the _repository ID_. The repository ID is internal to Gitaly Cluster and doesn't relate to any IDs elsewhere in GitLab. If a repository is removed from Gitaly Cluster and later moved @@ -525,12 +546,9 @@ To upgrade a Gitaly Cluster, follow the documentation for ### Downgrade Gitaly Cluster to a previous version -If you need to roll back a Gitaly Cluster to an earlier version, some Praefect database migrations may need to be reverted. In a cluster with: +If you need to roll back a Gitaly Cluster to an earlier version, some Praefect database migrations may need to be reverted. -- A single Praefect node, this happens when GitLab itself is downgraded. -- Multiple Praefect nodes, additional steps are required. - -To downgrade a Gitaly Cluster with multiple Praefect nodes: +To downgrade a Gitaly Cluster (assuming multiple Praefect nodes): 1. Stop the Praefect service on all Praefect nodes: @@ -565,7 +583,7 @@ To downgrade a Gitaly Cluster with multiple Praefect nodes: gitlab-ctl start praefect ``` -## Migrate to Gitaly Cluster +### Migrate to Gitaly Cluster WARNING: Some [known issues](#known-issues) exist in Gitaly Cluster. Review the following information before you continue. @@ -587,7 +605,7 @@ To migrate to Gitaly Cluster: Even if you don't use the `default` repository storage, you must ensure it is configured. [Read more about this limitation](configure_gitaly.md#gitlab-requires-a-default-repository-storage). -## Migrate off Gitaly Cluster +### Migrate off Gitaly Cluster If the limitations and tradeoffs of Gitaly Cluster are found to be not suitable for your environment, you can Migrate off Gitaly Cluster to a sharded Gitaly instance: @@ -596,22 +614,6 @@ off Gitaly Cluster to a sharded Gitaly instance: 1. [Move the repositories](../operations/moving_repositories.md#move-repositories) to the newly created storage. You can move them by shard or by group, which gives you the opportunity to spread them over multiple Gitaly servers. -## Do not bypass Gitaly - -GitLab doesn't advise directly accessing Gitaly repositories stored on disk with a Git client, -because Gitaly is being continuously improved and changed. These improvements may invalidate -your assumptions, resulting in performance degradation, instability, and even data loss. For example: - -- Gitaly has optimizations such as the [`info/refs` advertisement cache](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/design_diskcache.md), - that rely on Gitaly controlling and monitoring access to repositories by using the official gRPC - interface. -- [Gitaly Cluster](#gitaly-cluster) has optimizations, such as fault tolerance and - [distributed reads](#distributed-reads), that depend on the gRPC interface and database - to determine repository state. - -WARNING: -Accessing Git repositories directly is done at your own risk and is not supported. - ## Direct access to Git in GitLab Direct access to Git uses code in GitLab known as the "Rugged patches". @@ -683,8 +685,12 @@ To see if GitLab can access the repository file system directly, we use the foll - GitLab Rails tries to read the metadata file directly. If it exists, and if the UUID's match, assume we have direct access. -Direct Git access is enable by default in Omnibus GitLab because it fills in the correct repository -paths in the GitLab configuration file `config/gitlab.yml`. This satisfies the UUID check. +Versions of GitLab 15.3 and later disable direct Git access by default. + +For versions of GitLab prior to 15.3, direct Git access is enabled by +default in Omnibus GitLab because it fills in the correct repository +paths in the GitLab configuration file `config/gitlab.yml`. This +satisfies the UUID check. ### Transition to Gitaly Cluster diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md index 0dd9e0d7128..a435fff74fc 100644 --- a/doc/administration/gitaly/monitoring.md +++ b/doc/administration/gitaly/monitoring.md @@ -34,7 +34,7 @@ of requests dropped due to request limiting. The `reason` label indicates why a You can observe specific behavior of [concurrency-queued requests](configure_gitaly.md#limit-rpc-concurrency) using the Gitaly logs and Prometheus: -- In the [Gitaly logs](../logs.md#gitaly-logs), look for the string (or structured log field) +- In the [Gitaly logs](../logs/index.md#gitaly-logs), look for the string (or structured log field) `acquire_ms`. Messages that have this field are reporting about the concurrency limiter. - In Prometheus, look for the following metrics: - `gitaly_concurrency_limiting_in_progress` indicates how many concurrent requests are @@ -188,7 +188,7 @@ To monitor [repository verification](praefect.md#repository-verification), use t - `gitaly_praefect_stale_verification_leases_released_total`, the number of stale verification leases released. -You can also monitor the [Praefect logs](../logs.md#praefect-logs). +You can also monitor the [Praefect logs](../logs/index.md#praefect-logs). ### Database metrics `/db_metrics` endpoint diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 5635898293b..488e6a0df5f 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -97,8 +97,8 @@ If you [installed](https://about.gitlab.com/install/) GitLab using the Omnibus G ### Preparation -Before beginning, you should already have a working GitLab instance. [Learn how -to install GitLab](https://about.gitlab.com/install/). +Before beginning, you should already have a working GitLab instance. +[Learn how to install GitLab](https://about.gitlab.com/install/). Provision a PostgreSQL server. We recommend using the PostgreSQL that is shipped with Omnibus GitLab and use it to configure the PostgreSQL database. You can use an @@ -331,8 +331,8 @@ To configure the additional connection, you must either: #### Configure a new PgBouncer database with `pool_mode = session` We recommend using PgBouncer with `session` pool mode. You can use the -[bundled PgBouncer](../postgresql/pgbouncer.md) or use an external PgBouncer and [configure it -manually](https://www.pgbouncer.org/config.html). +[bundled PgBouncer](../postgresql/pgbouncer.md) or use an external PgBouncer and +[configure it manually](https://www.pgbouncer.org/config.html). The following example uses the bundled PgBouncer and sets up two separate connection pools on PostgreSQL host, one in `session` pool mode and the other in `transaction` pool mode. For this example to work, @@ -620,8 +620,8 @@ Updates to example must be made at: gitlab-ctl reconfigure ``` -1. To ensure that Praefect [has updated its Prometheus listen - address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), +1. To ensure that Praefect + [has updated its Prometheus listen address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart Praefect](../restart_gitlab.md#omnibus-gitlab-restart): ```shell @@ -928,8 +928,8 @@ For more information on Gitaly server configuration, see our gitlab-ctl reconfigure ``` -1. To ensure that Gitaly [has updated its Prometheus listen - address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), +1. To ensure that Gitaly + [has updated its Prometheus listen address](https://gitlab.com/gitlab-org/gitaly/-/issues/2734), [restart Gitaly](../restart_gitlab.md#omnibus-gitlab-restart): ```shell @@ -1103,10 +1103,8 @@ Particular attention should be shown to: ``` 1. Verify on each Gitaly node the Git Hooks can reach GitLab. On each Gitaly node run: - - ```shell - /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. 1. Verify that GitLab can reach Praefect: @@ -1151,8 +1149,7 @@ running multiple Gitaly storages. ### Grafana Grafana is included with GitLab, and can be used to monitor your Praefect -cluster. See [Grafana Dashboard -Service](https://docs.gitlab.com/omnibus/settings/grafana.html) +cluster. See [Grafana Dashboard Service](https://docs.gitlab.com/omnibus/settings/grafana.html) for detailed documentation. To get started quickly: @@ -1320,8 +1317,7 @@ praefect['background_verification_verification_interval'] = '0' WARNING: Deletions are disabled by default due to a race condition with repository renames that can cause incorrect deletions. This is especially prominent in Geo instances as Geo performs more renames than instances without Geo. -See [Handle repository creations, deletions and renames atomically](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4101) -for progress on a fix. We do not recommend enabling the deletions until this is fixed. +You should enable deletions only if the [`gitaly_praefect_generated_replica_paths` feature flag](index.md#praefect-generated-replica-paths-gitlab-150-and-later) is enabled. By default, the worker does not delete invalid metadata records but simply logs them and outputs Prometheus metrics for them. diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 551a7ab289a..bd4846a986d 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -296,6 +296,7 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t > - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3767) in GitLab 14.3. > - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4054) in GitLab 14.6, support for dry-run mode. +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4715) in GitLab 15.3, support for removing repositories from Praefect's database. The `remove-repository` Praefect sub-command removes a repository from a Gitaly Cluster, and all state associated with a given repository including: @@ -310,6 +311,9 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t - Replace `<virtual-storage>` with the name of the virtual storage containing the repository. - Replace `<repository>` with the relative path of the repository to remove. +- In GitLab 15.3 and later, add `-db-only` to remove the Praefect database entry without removing the on-disk repository. Use this option to remove orphaned database entries and to + protect on-disk repository data from deletion when a valid repository is accidentally specified. If the database entry is accidentally deleted, re-track the repository with the + [`track-repository` command](#manually-track-repositories). - In GitLab 14.6 and later, add `-apply` to run the command outside of dry-run mode and remove the repository. For example: ```shell diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index fb3a31d61b8..e308fa9da43 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -13,7 +13,7 @@ Refer to the information below when troubleshooting Gitaly and Gitaly Cluster. The following sections provide possible solutions to Gitaly errors. See also [Gitaly timeout](../../user/admin_area/settings/gitaly_timeouts.md) settings, -and our advice on [parsing the `gitaly/current` file](../troubleshooting/log_parsing.md#parsing-gitalycurrent). +and our advice on [parsing the `gitaly/current` file](../logs/log_parsing.md#parsing-gitalycurrent). ### Check versions when using standalone Gitaly servers @@ -24,6 +24,17 @@ as GitLab to ensure full compatibility: 1. On the left sidebar, select **Overview > Gitaly Servers**. 1. Confirm all Gitaly servers indicate that they are up to date. +### Find storage resource details + +You can run the following commands in a [Rails console](../operations/rails_console.md#starting-a-rails-console-session) +to determine the available and used space on a Gitaly storage: + +```ruby +Gitlab::GitalyClient::ServerService.new("default").storage_disk_statistics +# For Gitaly Cluster +Gitlab::GitalyClient::ServerService.new("<storage name>").disk_statistics +``` + ### Use `gitaly-debug` The `gitaly-debug` command provides "production debugging" tools for Gitaly and Git @@ -444,15 +455,15 @@ If you receive an error, check `/var/log/gitlab/gitlab-rails/production.log`. Here are common errors and potential causes: - 500 response code - - **ActionView::Template::Error (7:permission denied)** + - `ActionView::Template::Error (7:permission denied)` - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - - **Unable to save project. Error: 7:permission denied** + - `Unable to save project. Error: 7:permission denied` - Secret token in `praefect['storage_nodes']` on GitLab server does not match the value in `gitaly['auth_token']` on one or more Gitaly servers. - 503 response code - - **GRPC::Unavailable (14:failed to connect to all addresses)** + - `GRPC::Unavailable (14:failed to connect to all addresses)` - GitLab was unable to reach Praefect. - - **GRPC::Unavailable (14:all SubCons are in TransientFailure...)** + - `GRPC::Unavailable (14:all SubCons are in TransientFailure...)` - Praefect cannot reach one or more of its child Gitaly nodes. Try running the Praefect connection checker to diagnose. diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 95301ec026a..9351a101ee4 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -75,11 +75,12 @@ Email is processed correctly when a configured email address is present in one o (sorted in the order they are checked): - `To` -- `References` - `Delivered-To` - `Envelope-To` or `X-Envelope-To` - `Received` +The `References` header is also accepted, however it is used specifically to relate email responses to existing discussion threads. It is not used for creating issues by email. + In GitLab 14.6 and later, [Service Desk](../user/project/service_desk.md) also checks accepted headers. diff --git a/doc/administration/index.md b/doc/administration/index.md index 4565c0a8e00..3f2ae3170ab 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -207,9 +207,8 @@ Learn how to install, configure, update, and maintain your GitLab instance. ## Troubleshooting -- [Log system](logs.md): Where to look for logs. +- [Log system](logs/index.md): Where to look for logs. - [Sidekiq Troubleshooting](troubleshooting/sidekiq.md): Debug when Sidekiq appears hung and is not processing jobs. -- [Troubleshooting Elasticsearch](troubleshooting/elasticsearch.md) - [Navigating GitLab via Rails console](operations/rails_console.md) - [GitLab application limits](instance_limits.md) - [Responding to security incidents](../security/responding_to_security_incidents.md) @@ -241,4 +240,4 @@ who are aware of the risks. - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/testing-with-openssl/index.html) - [`strace` zine](https://wizardzines.com/zines/strace/) - GitLab.com-specific resources: - - [Group SAML/SCIM setup](troubleshooting/group_saml_scim.md) + - [Example group SAML and SCIM configurations](../user/group/saml_sso/example_saml_config.md) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 5158e2bbdd9..31434a8ee6f 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -639,6 +639,7 @@ setting is used: | `ci_max_artifact_size_secret_detection` | 0 | | `ci_max_artifact_size_terraform` | 5 MB ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37018) in GitLab 13.3) | | `ci_max_artifact_size_trace` | 0 | +| `ci_max_artifact_size_cyclonedx` | 1 MB | For example, to set the `ci_max_artifact_size_junit` limit to 10 MB on a self-managed installation, run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): @@ -1047,7 +1048,8 @@ The [secure files API](../api/secure_files.md) enforces the following limits: ## Changelog API limits -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89032) in GitLab 15.1 [with a flag](../administration/feature_flags.md) named `changelog_commits_limitation`. Disabled by default. +> - [Enabled on GitLab.com and by default on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/33893) in GitLab 15.3. The [changelog API](../api/repositories.md#add-changelog-data-to-a-changelog-file) enforces the following limits: diff --git a/doc/administration/integration/mailgun.md b/doc/administration/integration/mailgun.md index 6b0cb0466fc..c007116d213 100644 --- a/doc/administration/integration/mailgun.md +++ b/doc/administration/integration/mailgun.md @@ -18,8 +18,8 @@ After completing the integration, Mailgun `temporary_failure` and `permanent_fai ## Configure your Mailgun domain -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the `/-/members/mailgun/permanent_failures` URL in GitLab 15.0. -> [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the URL to handle both temporary and permanent failures in GitLab 15.0. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the `/-/members/mailgun/permanent_failures` URL in GitLab 15.0. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/359113) the URL to handle both temporary and permanent failures in GitLab 15.0. Before you can enable Mailgun in GitLab, set up your own Mailgun endpoints to receive the webhooks. @@ -47,6 +47,6 @@ you're ready to enable the Mailgun integration: 1. On the left sidebar, go to **Settings > General** and expand the **Mailgun** section. 1. Select the **Enable Mailgun** check box. 1. Enter the Mailgun HTTP webhook signing key as described in - [the Mailgun documentation](https://documentation.mailgun.com/en/latest/user_manual.html#webhooks) and + [the Mailgun documentation](https://documentation.mailgun.com/en/latest/user_manual.html#webhooks-1) and shown in the [API security](https://app.mailgun.com/app/account/security/api_keys) section for your Mailgun account. 1. Select **Save changes**. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 2582df1b0c4..a4ab0e07020 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -533,7 +533,7 @@ review: WARNING: Uploading artifacts as "archive" to coordinator... failed id=12345 responseStatus=500 Internal Server Error status=500 token=abcd1234 ``` -- The [workhorse log](logs.md#workhorse-logs) for an error message similar to: +- The [workhorse log](logs/index.md#workhorse-logs) for an error message similar to: ```json {"error":"MissingRegion: could not find region configuration","level":"error","msg":"error uploading S3 session","time":"2021-03-16T22:10:55-04:00"} diff --git a/doc/administration/libravatar.md b/doc/administration/libravatar.md index eac7c6f848b..88a46759924 100644 --- a/doc/administration/libravatar.md +++ b/doc/administration/libravatar.md @@ -11,63 +11,99 @@ GitLab by default supports the [Gravatar](https://gravatar.com) avatar service. Libravatar is another service that delivers your avatar (profile picture) to other websites. The Libravatar API is -[heavily based on gravatar](https://wiki.libravatar.org/api/), so you can +[heavily based on Gravatar](https://wiki.libravatar.org/api/), so you can switch to the Libravatar avatar service or even your own Libravatar server. -## Configuration +You cannot use any Libravatar service including Gravatar in [FIPS mode](../development/fips_compliance.md). -In the [`gitlab.yml` gravatar section](https://gitlab.com/gitlab-org/gitlab/-/blob/672bd3902d86b78d730cea809fce312ec49d39d7/config/gitlab.yml.example#L122), set +## Change the Libravatar service to your own service + +NOTE: +You can use only the MD5 hash in the URL for the Libravatar service. See [issue 370057](https://gitlab.com/gitlab-org/gitlab/-/issues/370057) for adding SHA-256 support. + +In the [`gitlab.yml` gravatar section](https://gitlab.com/gitlab-org/gitlab/-/blob/68dac188ec6b1b03d53365e7579422f44cbe7a1c/config/gitlab.yml.example#L469-476), set the configuration options as follows: -### For HTTP +**For Omnibus installations** -```yaml - gravatar: - enabled: true - # gravatar URLs: possible placeholders: %{hash} %{size} %{email} %{username} - plain_url: "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" -``` +1. Edit `/etc/gitlab/gitlab.rb`: -### For HTTPS + ```ruby + gitlab_rails['gravatar_enabled'] = true + #### For HTTPS + gitlab_rails['gravatar_ssl_url'] = "https://seccdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" + #### Use this line instead for HTTP + # gitlab_rails['gravatar_plain_url'] = "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" + ``` -```yaml - gravatar: - enabled: true - # gravatar URLs: possible placeholders: %{hash} %{size} %{email} %{username} - ssl_url: "https://seccdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" -``` +1. To apply the changes, run `sudo gitlab-ctl reconfigure`. -### Your own Libravatar server +**For installations from source** -If you are [running your own Libravatar service](https://wiki.libravatar.org/running_your_own/), -the URL is different in the configuration, but you must provide the same -placeholders so GitLab can parse the URL correctly. +1. Edit `config/gitlab.yml`: -For example, you host a service on `http://libravatar.example.com` and the -`plain_url` you must supply in `gitlab.yml` is + ```yaml + gravatar: + enabled: true + # default: https://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + plain_url: "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" + # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon + ssl_url: https://seccdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" + ``` -`http://libravatar.example.com/avatar/%{hash}?s=%{size}&d=identicon` +1. Save the file, and then [restart](restart_gitlab.md#installations-from-source) + GitLab for the changes to take effect. -### Omnibus GitLab example +## Set the Libravatar service to default (Gravatar) -In `/etc/gitlab/gitlab.rb`: +**For Omnibus installations** -#### For HTTP +1. Delete `gitlab_rails['gravatar_ssl_url']` or `gitlab_rails['gravatar_plain_url']` from `/etc/gitlab/gitlab.rb`. +1. To apply the changes, run `sudo gitlab-ctl reconfigure`. -```ruby -gitlab_rails['gravatar_enabled'] = true -gitlab_rails['gravatar_plain_url'] = "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" -``` +**For installations from source** -#### For HTTPS +1. Remove `gravatar:` section from `config/gitlab.yml`. +1. Save the file, then [restart](restart_gitlab.md#installations-from-source) + GitLab to apply the changes. -```ruby -gitlab_rails['gravatar_enabled'] = true -gitlab_rails['gravatar_ssl_url'] = "https://seccdn.libravatar.org/avatar/%{hash}?s=%{size}&d=identicon" -``` +## Disable Gravatar service + +To disable Gravatar, for example, to prohibit third-party services, complete the following steps: + +**For Omnibus installations** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['gravatar_enabled'] = false + ``` + +1. To apply the changes, run `sudo gitlab-ctl reconfigure`. + +**For installations from source** + +1. Edit `config/gitlab.yml`: + + ```yaml + gravatar: + enabled: false + ``` + +1. Save the file, then [restart](restart_gitlab.md#installations-from-source) + GitLab to apply the changes. + +### Your own Libravatar server + +If you are [running your own Libravatar service](https://wiki.libravatar.org/running_your_own/), +the URL is different in the configuration, but you must provide the same +placeholders so GitLab can parse the URL correctly. + +For example, you host a service on `https://libravatar.example.com` and the +`ssl_url` you must supply in `gitlab.yml` is: -Then run `sudo gitlab-ctl reconfigure` for the changes to take effect. +`https://libravatar.example.com/avatar/%{hash}?s=%{size}&d=identicon` ## Default URL for missing images @@ -77,7 +113,7 @@ service. To use a set other than `identicon`, replace the `&d=identicon` portion of the URL with another supported set. For example, you can use the `retro` set, in -which case the URL would look like: `plain_url: "http://cdn.libravatar.org/avatar/%{hash}?s=%{size}&d=retro"` +which case the URL would look like: `ssl_url: "https://seccdn.libravatar.org/avatar/%{hash}?s=%{size}&d=retro"` ## Usage examples for Microsoft Office 365 diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 671c264ed85..7264cf6db98 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -1,1178 +1,11 @@ --- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'logs/index.md' +remove_date: '2022-10-23' --- -# Log system **(FREE SELF)** +This document was moved to [another location](logs/index.md). -GitLab has an advanced log system where everything is logged, so you -can analyze your instance using various system log files. In addition to -system log files, GitLab Enterprise Edition provides [Audit Events](audit_events.md). - -System log files are typically plain text in a standard log file format. -This guide talks about how to read and use these system log files. - -Read more about the log system and using the logs: - -- [Customize logging on Omnibus GitLab installations](https://docs.gitlab.com/omnibus/settings/logs.html) -including adjusting log retention, log forwarding, -switching logs from JSON to plain text logging, and more. -- [How to parse and analyze JSON logs](troubleshooting/log_parsing.md). - -## Log Levels - -Each log message has an assigned log level that indicates its importance and verbosity. -Each logger has an assigned minimum log level. -A logger emits a log message only if its log level is equal to or above the minimum log level. - -The following log levels are supported: - -| Level | Name | -|:------|:----------| -| 0 | `DEBUG` | -| 1 | `INFO` | -| 2 | `WARN` | -| 3 | `ERROR` | -| 4 | `FATAL` | -| 5 | `UNKNOWN` | - -GitLab loggers emit all log messages because they are set to `DEBUG` by default. - -### Override default log level - -You can override the minimum log level for GitLab loggers using the `GITLAB_LOG_LEVEL` environment variable. -Valid values are either a value of `0` to `5`, or the name of the log level. - -Example: - -```shell -GITLAB_LOG_LEVEL=info -``` - -For some services, other log levels are in place that are not affected by this setting. -Some of these services have their own environment variables to override the log level. For example: - -| Service | Log level | Environment variable | -|:---------------------|:----------|:---------------------| -| GitLab API | `INFO` | | -| GitLab Cleanup | `INFO` | `DEBUG` | -| GitLab Doctor | `INFO` | `VERBOSE` | -| GitLab Export | `INFO` | `EXPORT_DEBUG` | -| GitLab Geo | `INFO` | | -| GitLab Import | `INFO` | `IMPORT_DEBUG` | -| GitLab QA Runtime | `INFO` | `QA_LOG_LEVEL` | -| Google APIs | `INFO` | | -| Rack Timeout | `ERROR` | | -| Sidekiq (server) | `INFO` | | -| Snowplow Tracker | `FATAL` | | -| gRPC Client (Gitaly) | `WARN` | `GRPC_LOG_LEVEL` | - -## Log Rotation - -The logs for a given service may be managed and rotated by: - -- `logrotate` -- `svlogd` (`runit`'s service logging daemon) -- `logrotate` and `svlogd` -- Or not at all - -The following table includes information about what's responsible for managing and rotating logs for -the included services. Logs -[managed by `svlogd`](https://docs.gitlab.com/omnibus/settings/logs.html#runit-logs) -are written to a file called `current`. The `logrotate` service built into GitLab -[manages all logs](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate) -except those captured by `runit`. - -| Log type | Managed by logrotate | Managed by svlogd/runit | -|:------------------------------------------------|:------------------------|:------------------------| -| [Alertmanager Logs](#alertmanager-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Crond Logs](#crond-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Gitaly](#gitaly-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| [GitLab Exporter for Omnibus](#gitlab-exporter) | **{dotted-circle}** No | **{check-circle}** Yes | -| [GitLab Pages Logs](#pages-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| GitLab Rails | **{check-circle}** Yes | **{dotted-circle}** No | -| [GitLab Shell Logs](#gitlab-shelllog) | **{check-circle}** Yes | **{dotted-circle}** No | -| [Grafana Logs](#grafana-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [LogRotate Logs](#logrotate-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Mailroom](#mail_room_jsonlog-default) | **{check-circle}** Yes | **{check-circle}** Yes | -| [NGINX](#nginx-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| [PostgreSQL Logs](#postgresql-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Praefect Logs](#praefect-logs) | **{dotted-circle}** Yes | **{check-circle}** Yes | -| [Prometheus Logs](#prometheus-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Puma](#puma-logs) | **{check-circle}** Yes | **{check-circle}** Yes | -| [Redis Logs](#redis-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Registry Logs](#registry-logs) | **{dotted-circle}** No | **{check-circle}** Yes | -| [Workhorse Logs](#workhorse-logs) | **{check-circle}** Yes | **{check-circle}** Yes | - -## `production_json.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/production_json.log` -- Installations from source: `/home/git/gitlab/log/production_json.log` - -It contains a structured log for Rails controller requests received from -GitLab, thanks to [Lograge](https://github.com/roidrage/lograge/). -Requests from the API are logged to a separate file in `api_json.log`. - -Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. -Line breaks were added to examples for legibility: - -```json -{ - "method":"GET", - "path":"/gitlab/gitlab-foss/issues/1234", - "format":"html", - "controller":"Projects::IssuesController", - "action":"show", - "status":200, - "time":"2017-08-08T20:15:54.821Z", - "params":[{"key":"param_key","value":"param_value"}], - "remote_ip":"18.245.0.1", - "user_id":1, - "username":"admin", - "queue_duration_s":0.0, - "gitaly_calls":16, - "gitaly_duration_s":0.16, - "redis_calls":115, - "redis_duration_s":0.13, - "redis_read_bytes":1507378, - "redis_write_bytes":2920, - "correlation_id":"O1SdybnnIq7", - "cpu_s":17.50, - "db_duration_s":0.08, - "view_duration_s":2.39, - "duration_s":20.54, - "pid": 81836, - "worker_id":"puma_0" -} -``` - -This example was a GET request for a specific -issue. Each line also contains performance data, with times in -seconds: - -- `duration_s`: Total time to retrieve the request -- `queue_duration_s`: Total time the request was queued inside GitLab Workhorse -- `view_duration_s`: Total time inside the Rails views -- `db_duration_s`: Total time to retrieve data from PostgreSQL -- `cpu_s`: Total time spent on CPU -- `gitaly_duration_s`: Total time by Gitaly calls -- `gitaly_calls`: Total number of calls made to Gitaly -- `redis_calls`: Total number of calls made to Redis -- `redis_duration_s`: Total time to retrieve data from Redis -- `redis_read_bytes`: Total bytes read from Redis -- `redis_write_bytes`: Total bytes written to Redis -- `redis_<instance>_calls`: Total number of calls made to a Redis instance -- `redis_<instance>_duration_s`: Total time to retrieve data from a Redis instance -- `redis_<instance>_read_bytes`: Total bytes read from a Redis instance -- `redis_<instance>_write_bytes`: Total bytes written to a Redis instance -- `pid`: The worker's Linux process ID (changes when workers restart) -- `worker_id`: The worker's logical ID (does not change when workers restart) - -User clone and fetch activity using HTTP transport appears in the log as `action: git_upload_pack`. - -In addition, the log contains the originating IP address, -(`remote_ip`), the user's ID (`user_id`), and username (`username`). - -Some endpoints (such as `/search`) may make requests to Elasticsearch if using -[Advanced Search](../user/search/advanced_search.md). These -additionally log `elasticsearch_calls` and `elasticsearch_call_duration_s`, -which correspond to: - -- `elasticsearch_calls`: Total number of calls to Elasticsearch -- `elasticsearch_duration_s`: Total time taken by Elasticsearch calls -- `elasticsearch_timed_out_count`: Total number of calls to Elasticsearch that - timed out and therefore returned partial results - -ActionCable connection and subscription events are also logged to this file and they follow the -previous format. The `method`, `path`, and `format` fields are not applicable, and are always empty. -The ActionCable connection or channel class is used as the `controller`. - -```json -{ - "method":null, - "path":null, - "format":null, - "controller":"IssuesChannel", - "action":"subscribe", - "status":200, - "time":"2020-05-14T19:46:22.008Z", - "params":[{"key":"project_path","value":"gitlab/gitlab-foss"},{"key":"iid","value":"1"}], - "remote_ip":"127.0.0.1", - "user_id":1, - "username":"admin", - "ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0", - "correlation_id":"jSOIEynHCUa", - "duration_s":0.32566 -} -``` - -NOTE: -Starting with GitLab 12.5, if an error occurs, an -`exception` field is included with `class`, `message`, and -`backtrace`. Previous versions included an `error` field instead of -`exception.class` and `exception.message`. For example: - -```json -{ - "method": "GET", - "path": "/admin", - "format": "html", - "controller": "Admin::DashboardController", - "action": "index", - "status": 500, - "time": "2019-11-14T13:12:46.156Z", - "params": [], - "remote_ip": "127.0.0.1", - "user_id": 1, - "username": "root", - "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:70.0) Gecko/20100101 Firefox/70.0", - "queue_duration": 274.35, - "correlation_id": "KjDVUhNvvV3", - "queue_duration_s":0.0, - "gitaly_calls":16, - "gitaly_duration_s":0.16, - "redis_calls":115, - "redis_duration_s":0.13, - "correlation_id":"O1SdybnnIq7", - "cpu_s":17.50, - "db_duration_s":0.08, - "view_duration_s":2.39, - "duration_s":20.54, - "pid": 81836, - "worker_id": "puma_0", - "exception.class": "NameError", - "exception.message": "undefined local variable or method `adsf' for #<Admin::DashboardController:0x00007ff3c9648588>", - "exception.backtrace": [ - "app/controllers/admin/dashboard_controller.rb:11:in `index'", - "ee/app/controllers/ee/admin/dashboard_controller.rb:14:in `index'", - "ee/lib/gitlab/ip_address_state.rb:10:in `with'", - "ee/app/controllers/ee/application_controller.rb:43:in `set_current_ip_address'", - "lib/gitlab/session.rb:11:in `with_session'", - "app/controllers/application_controller.rb:450:in `set_session_storage'", - "app/controllers/application_controller.rb:444:in `set_locale'", - "ee/lib/gitlab/jira/middleware.rb:19:in `call'" - ] -} -``` - -## `production.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/production.log` -- Installations from source: `/home/git/gitlab/log/production.log` - -It contains information about all performed requests. You can see the -URL and type of request, IP address, and what parts of code were -involved to service this particular request. Also, you can see all SQL -requests performed, and how much time each took. This task is -more useful for GitLab contributors and developers. Use part of this log -file when you're reporting bugs. For example: - -```plaintext -Started GET "/gitlabhq/yaml_db/tree/master" for 168.111.56.1 at 2015-02-12 19:34:53 +0200 -Processing by Projects::TreeController#show as HTML - Parameters: {"project_id"=>"gitlabhq/yaml_db", "id"=>"master"} - - ... [CUT OUT] - - Namespaces"."created_at" DESC, "namespaces"."id" DESC LIMIT 1 [["id", 26]] - CACHE (0.0ms) SELECT "members".* FROM "members" WHERE "members"."source_type" = 'Project' AND "members"."type" IN ('ProjectMember') AND "members"."source_id" = $1 AND "members"."source_type" = $2 AND "members"."user_id" = 1 ORDER BY "members"."created_at" DESC, "members"."id" DESC LIMIT 1 [["source_id", 18], ["source_type", "Project"]] - CACHE (0.0ms) SELECT "members".* FROM "members" WHERE "members"."source_type" = 'Project' AND "members". - (1.4ms) SELECT COUNT(*) FROM "merge_requests" WHERE "merge_requests"."target_project_id" = $1 AND ("merge_requests"."state" IN ('opened','reopened')) [["target_project_id", 18]] - Rendered layouts/nav/_project.html.haml (28.0ms) - Rendered layouts/_collapse_button.html.haml (0.2ms) - Rendered layouts/_flash.html.haml (0.1ms) - Rendered layouts/_page.html.haml (32.9ms) -Completed 200 OK in 166ms (Views: 117.4ms | ActiveRecord: 27.2ms) -``` - -In this example, the server processed an HTTP request with URL -`/gitlabhq/yaml_db/tree/master` from IP `168.111.56.1` at `2015-02-12 19:34:53 +0200`. -The request was processed by `Projects::TreeController`. - -## `api_json.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/api_json.log` -- Installations from source: `/home/git/gitlab/log/api_json.log` - -It helps you see requests made directly to the API. For example: - -```json -{ - "time":"2018-10-29T12:49:42.123Z", - "severity":"INFO", - "duration":709.08, - "db":14.59, - "view":694.49, - "status":200, - "method":"GET", - "path":"/api/v4/projects", - "params":[{"key":"action","value":"git-upload-pack"},{"key":"changes","value":"_any"},{"key":"key_id","value":"secret"},{"key":"secret_token","value":"[FILTERED]"}], - "host":"localhost", - "remote_ip":"::1", - "ua":"Ruby", - "route":"/api/:version/projects", - "user_id":1, - "username":"root", - "queue_duration":100.31, - "gitaly_calls":30, - "gitaly_duration":5.36, - "pid": 81836, - "worker_id": "puma_0", - ... -} -``` - -This entry shows an internal endpoint accessed to check whether an -associated SSH key can download the project in question by using a `git fetch` or -`git clone`. In this example, we see: - -- `duration`: Total time in milliseconds to retrieve the request -- `queue_duration`: Total time in milliseconds the request was queued inside GitLab Workhorse -- `method`: The HTTP method used to make the request -- `path`: The relative path of the query -- `params`: Key-value pairs passed in a query string or HTTP body (sensitive parameters, such as passwords and tokens, are filtered out) -- `ua`: The User-Agent of the requester - -## `application.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application.log` -- Installations from source: `/home/git/gitlab/log/application.log` - -It helps you discover events happening in your instance such as user creation -and project deletion. For example: - -```plaintext -October 06, 2014 11:56: User "Administrator" (admin@example.com) was created -October 06, 2014 11:56: Documentcloud created a new project "Documentcloud / Underscore" -October 06, 2014 11:56: Gitlab Org created a new project "Gitlab Org / Gitlab Ce" -October 07, 2014 11:25: User "Claudie Hodkiewicz" (nasir_stehr@olson.co.uk) was removed -October 07, 2014 11:25: Project "project133" was removed -``` - -## `application_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22812) in GitLab 12.7. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application_json.log` -- Installations from source: `/home/git/gitlab/log/application_json.log` - -It contains the JSON version of the logs in `application.log`, like this example: - -```json -{ - "severity":"INFO", - "time":"2020-01-14T13:35:15.466Z", - "correlation_id":"3823a1550b64417f9c9ed8ee0f48087e", - "message":"User \"Administrator\" (admin@example.com) was created" -} -{ - "severity":"INFO", - "time":"2020-01-14T13:35:15.466Z", - "correlation_id":"78e3df10c9a18745243d524540bd5be4", - "message":"Project \"project133\" was removed" -} -``` - -## `integrations_json.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/integrations_json.log` -- Installations from source: `/home/git/gitlab/log/integrations_json.log` - -It contains information about [integration](../user/project/integrations/index.md) -activities, such as Jira, Asana, and irker services. It uses JSON format, -like this example: - -```json -{ - "severity":"ERROR", - "time":"2018-09-06T14:56:20.439Z", - "service_class":"Integrations::Jira", - "project_id":8, - "project_path":"h5bp/html5-boilerplate", - "message":"Error sending message", - "client_url":"http://jira.gitlap.com:8080", - "error":"execution expired" -} -{ - "severity":"INFO", - "time":"2018-09-06T17:15:16.365Z", - "service_class":"Integrations::Jira", - "project_id":3, - "project_path":"namespace2/project2", - "message":"Successfully posted", - "client_url":"http://jira.example.com" -} -``` - -## `kubernetes.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/kubernetes.log` -- Installations from source: `/home/git/gitlab/log/kubernetes.log` - -It logs information related to the Kubernetes Integration, including errors -during installing cluster applications on your managed Kubernetes -clusters. - -Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. -Line breaks have been added to the following example for clarity: - -```json -{ - "severity":"ERROR", - "time":"2018-11-23T15:14:54.652Z", - "exception":"Kubeclient::HttpError", - "error_code":401, - "service":"Clusters::Applications::CheckInstallationProgressService", - "app_id":14, - "project_ids":[1], - "group_ids":[], - "message":"Unauthorized" -} -{ - "severity":"ERROR", - "time":"2018-11-23T15:42:11.647Z", - "exception":"Kubeclient::HttpError", - "error_code":null, - "service":"Clusters::Applications::InstallService", - "app_id":2, - "project_ids":[19], - "group_ids":[], - "message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)" -} -``` - -## `git_json.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/git_json.log` -- Installations from source: `/home/git/gitlab/log/git_json.log` - -After GitLab version 12.2, this file was renamed from `githost.log` to -`git_json.log` and stored in JSON format. - -GitLab has to interact with Git repositories, but in some rare cases -something can go wrong. If this happens, you need to know exactly what -happened. This log file contains all failed requests from GitLab to Git -repositories. In the majority of cases this file is useful for developers -only. For example: - -```json -{ - "severity":"ERROR", - "time":"2019-07-19T22:16:12.528Z", - "correlation_id":"FeGxww5Hj64", - "message":"Command failed [1]: /usr/bin/git --git-dir=/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/group184/gitlabhq/.git --work-tree=/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/group184/gitlabhq merge --no-ff -mMerge branch 'feature_conflict' into 'feature' source/feature_conflict\n\nerror: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git'" -} -``` - -## `audit_json.log` **(FREE)** - -NOTE: -GitLab Free tracks a small number of different audit events. -GitLab Premium tracks many more. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/audit_json.log` -- Installations from source: `/home/git/gitlab/log/audit_json.log` - -Changes to group or project settings and memberships (`target_details`) -are logged to this file. For example: - -```json -{ - "severity":"INFO", - "time":"2018-10-17T17:38:22.523Z", - "author_id":3, - "entity_id":2, - "entity_type":"Project", - "change":"visibility", - "from":"Private", - "to":"Public", - "author_name":"John Doe4", - "target_id":2, - "target_type":"Project", - "target_details":"namespace2/project2" -} -``` - -## Sidekiq Logs - -NOTE: -In Omnibus GitLab `12.10` or earlier, the Sidekiq log is at `/var/log/gitlab/gitlab-rails/sidekiq.log`. - -For Omnibus GitLab installations, some Sidekiq logs are in `/var/log/gitlab/sidekiq/current` -and as follows. - -### `sidekiq.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/sidekiq/current` -- Installations from source: `/home/git/gitlab/log/sidekiq.log` - -GitLab uses background jobs for processing tasks which can take a long -time. All information about processing these jobs are written down to -this file. For example: - -```plaintext -2014-06-10T07:55:20Z 2037 TID-tm504 ERROR: /opt/bitnami/apps/discourse/htdocs/vendor/bundle/ruby/1.9.1/gems/redis-3.0.7/lib/redis/client.rb:228:in `read' -2014-06-10T18:18:26Z 14299 TID-55uqo INFO: Booting Sidekiq 3.0.0 with redis options {:url=>"redis://localhost:6379/0", :namespace=>"sidekiq"} -``` - -Instead of the previous format, you can opt to generate JSON logs for -Sidekiq. For example: - -```json -{ - "severity":"INFO", - "time":"2018-04-03T22:57:22.071Z", - "queue":"cronjob:update_all_mirrors", - "args":[], - "class":"UpdateAllMirrorsWorker", - "retry":false, - "queue_namespace":"cronjob", - "jid":"06aeaa3b0aadacf9981f368e", - "created_at":"2018-04-03T22:57:21.930Z", - "enqueued_at":"2018-04-03T22:57:21.931Z", - "pid":10077, - "worker_id":"sidekiq_0", - "message":"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", - "job_status":"done", - "duration":0.139, - "completed_at":"2018-04-03T22:57:22.071Z", - "db_duration":0.05, - "db_duration_s":0.0005, - "gitaly_duration":0, - "gitaly_calls":0 -} -``` - -For Omnibus GitLab installations, add the configuration option: - -```ruby -sidekiq['log_format'] = 'json' -``` - -For installations from source, edit the `gitlab.yml` and set the Sidekiq -`log_format` configuration option: - -```yaml - ## Sidekiq - sidekiq: - log_format: json -``` - -### `sidekiq_client.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26586) in GitLab 12.9. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/sidekiq_client.log` -- Installations from source: `/home/git/gitlab/log/sidekiq_client.log` - -This file contains logging information about jobs before Sidekiq starts -processing them, such as before being enqueued. - -This log file follows the same structure as -[`sidekiq.log`](#sidekiqlog), so it is structured as JSON if -you've configured this for Sidekiq as mentioned above. - -## `gitlab-shell.log` - -GitLab Shell is used by GitLab for executing Git commands and provide SSH -access to Git repositories. - -### For GitLab versions 12.10 and up - -Information containing `git-{upload-pack,receive-pack}` requests is at -`/var/log/gitlab/gitlab-shell/gitlab-shell.log`. Information about hooks to -GitLab Shell from Gitaly is at `/var/log/gitlab/gitaly/current`. - -Example log entries for `/var/log/gitlab/gitlab-shell/gitlab-shell.log`: - -```json -{ - "duration_ms": 74.104, - "level": "info", - "method": "POST", - "msg": "Finished HTTP request", - "time": "2020-04-17T20:28:46Z", - "url": "http://127.0.0.1:8080/api/v4/internal/allowed" -} -{ - "command": "git-upload-pack", - "git_protocol": "", - "gl_project_path": "root/example", - "gl_repository": "project-1", - "level": "info", - "msg": "executing git command", - "time": "2020-04-17T20:28:46Z", - "user_id": "user-1", - "username": "root" -} -``` - -Example log entries for `/var/log/gitlab/gitaly/current`: - -```json -{ - "method": "POST", - "url": "http://127.0.0.1:8080/api/v4/internal/allowed", - "duration": 0.058012959, - "gitaly_embedded": true, - "pid": 16636, - "level": "info", - "msg": "finished HTTP request", - "time": "2020-04-17T20:29:08+00:00" -} -{ - "method": "POST", - "url": "http://127.0.0.1:8080/api/v4/internal/pre_receive", - "duration": 0.031022552, - "gitaly_embedded": true, - "pid": 16636, - "level": "info", - "msg": "finished HTTP request", - "time": "2020-04-17T20:29:08+00:00" -} -``` - -### For GitLab versions 12.5 through 12.9 - -For GitLab 12.5 to 12.9, depending on your installation method, this -file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitaly/gitlab-shell.log` -- Installation from source: `/home/git/gitaly/gitlab-shell.log` - -Example log entries: - -```json -{ - "method": "POST", - "url": "http://127.0.0.1:8080/api/v4/internal/post_receive", - "duration": 0.031809164, - "gitaly_embedded": true, - "pid": 27056, - "level": "info", - "msg": "finished HTTP request", - "time": "2020-04-17T16:24:38+00:00" -} -``` - -### For GitLab 12.5 and earlier - -For GitLab 12.5 and earlier, the file is at `/var/log/gitlab/gitlab-shell/gitlab-shell.log`. - -Example log entries: - -```plaintext -I, [2015-02-13T06:17:00.671315 #9291] INFO -- : Adding project root/example.git at </var/opt/gitlab/git-data/repositories/root/dcdcdcdcd.git>. -I, [2015-02-13T06:17:00.679433 #9291] INFO -- : Moving existing hooks directory and symlinking global hooks directory for /var/opt/gitlab/git-data/repositories/root/example.git. -``` - -User clone/fetch activity using SSH transport appears in this log as -`executing git command <gitaly-upload-pack...`. - -## Gitaly Logs - -This file is in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). -`runit` is packaged with Omnibus GitLab and a brief explanation of its purpose -is available [in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/architecture/#runit). -[Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in -Unix timestamp format, and `gzip`-compressed (like `@1584057562.s`). - -### `grpc.log` - -This file is at `/var/log/gitlab/gitlab-rails/grpc.log` for Omnibus GitLab -packages. Native [gRPC](https://grpc.io/) logging used by Gitaly. - -### `gitaly_ruby_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2678) in GitLab 13.6. - -This file is at `/var/log/gitlab/gitaly/gitaly_ruby_json.log` and is -produced by [`gitaly-ruby`](gitaly/reference.md#gitaly-ruby). It contains an -access log of gRPC calls made by Gitaly to `gitaly-ruby`. - -### `gitaly_hooks.log` - -This file is at `/var/log/gitlab/gitaly/gitaly_hooks.log` and is -produced by `gitaly-hooks` command. It also contains records about -failures received during processing of the responses from GitLab API. - -## Puma Logs - -### `puma_stdout.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/puma/puma_stdout.log` -- Installations from source: `/home/git/gitlab/log/puma_stdout.log` - -### `puma_stderr.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/puma/puma_stderr.log` -- Installations from source: `/home/git/gitlab/log/puma_stderr.log` - -## `repocheck.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/repocheck.log` -- Installations from source: `/home/git/gitlab/log/repocheck.log` - -It logs information whenever a [repository check is run](repository_checks.md) -on a project. - -## `importer.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/importer.log` -- Installations from source: `/home/git/gitlab/log/importer.log` - -It logs the progress of the import process. - -## `exporter.log` - -> Introduced in GitLab 13.1. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/exporter.log` -- Installations from source: `/home/git/gitlab/log/exporter.log` - -It logs the progress of the export process. - -## `features_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 13.7. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/features_json.log` -- Installations from source: `/home/git/gitlab/log/features_json.log` - -The modification events from [Feature flags in development of GitLab](../development/feature_flags/index.md) -are recorded in this file. For example: - -```json -{"severity":"INFO","time":"2020-11-24T02:30:59.860Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"true"} -{"severity":"INFO","time":"2020-11-24T02:31:29.108Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"true"} -{"severity":"INFO","time":"2020-11-24T02:31:29.129Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable","extra.thing":"false"} -{"severity":"INFO","time":"2020-11-24T02:31:29.177Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"Project:1"} -{"severity":"INFO","time":"2020-11-24T02:31:29.183Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable","extra.thing":"Project:1"} -{"severity":"INFO","time":"2020-11-24T02:31:29.188Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable_percentage_of_time","extra.percentage":"50"} -{"severity":"INFO","time":"2020-11-24T02:31:29.193Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable_percentage_of_time"} -{"severity":"INFO","time":"2020-11-24T02:31:29.198Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable_percentage_of_actors","extra.percentage":"50"} -{"severity":"INFO","time":"2020-11-24T02:31:29.203Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable_percentage_of_actors"} -{"severity":"INFO","time":"2020-11-24T02:31:29.329Z","correlation_id":null,"key":"cd_auto_rollback","action":"remove"} -``` - -## `auth.log` - -> Introduced in GitLab 12.0. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/auth.log` -- Installations from source: `/home/git/gitlab/log/auth.log` - -This log records: - -- Requests over the [Rate Limit](../user/admin_area/settings/rate_limits_on_raw_endpoints.md) on raw endpoints. -- [Protected paths](../user/admin_area/settings/protected_paths.md) abusive requests. -- In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and later, - user ID and username, if available. - -## `graphql_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 12.0. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/graphql_json.log` -- Installations from source: `/home/git/gitlab/log/graphql_json.log` - -GraphQL queries are recorded in the file. For example: - -```json -{"query_string":"query IntrospectionQuery{__schema {queryType { name },mutationType { name }}}...(etc)","variables":{"a":1,"b":2},"complexity":181,"depth":1,"duration_s":7} -``` - -## `migrations.log` - -> Introduced in GitLab 12.3. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/migrations.log` -- Installations from source: `/home/git/gitlab/log/migrations.log` - -## `mail_room_json.log` (default) - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186) in GitLab 12.6. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/mailroom/current` -- Installations from source: `/home/git/gitlab/log/mail_room_json.log` - -This structured log file records internal activity in the `mail_room` gem. -Its name and path are configurable, so the name and path may not match the above. - -## Reconfigure logs - -Reconfigure log files are in `/var/log/gitlab/reconfigure` for Omnibus GitLab -packages. Installations from source don't have reconfigure logs. A reconfigure log -is populated whenever `gitlab-ctl reconfigure` is run manually or as part of an upgrade. - -Reconfigure logs files are named according to the UNIX timestamp of when the reconfigure -was initiated, such as `1509705644.log` - -## `sidekiq_exporter.log` and `web_exporter.log` - -If Prometheus metrics and the Sidekiq Exporter are both enabled, Sidekiq -starts a Web server and listen to the defined port (default: -`8082`). By default, Sidekiq Exporter access logs are disabled but can -be enabled based on your installation method: - -- Omnibus GitLab: Use the `sidekiq['exporter_log_enabled'] = true` - option in `/etc/gitlab/gitlab.rb` -- Installations from source: Use the `sidekiq_exporter.log_enabled` option - in `gitlab.yml` - -When enabled, depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/sidekiq_exporter.log` -- Installations from source: `/home/git/gitlab/log/sidekiq_exporter.log` - -If Prometheus metrics and the Web Exporter are both enabled, Puma -starts a Web server and listen to the defined port (default: `8083`), and access logs -are generated in a location based on your installation method: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/web_exporter.log` -- Installations from source: `/home/git/gitlab/log/web_exporter.log` - -## `database_load_balancing.log` **(PREMIUM SELF)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/15442) in GitLab 12.3. - -Contains details of GitLab [Database Load Balancing](postgresql/database_load_balancing.md). -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/database_load_balancing.log` -- Installations from source: `/home/git/gitlab/log/database_load_balancing.log` - -## `elasticsearch.log` **(PREMIUM SELF)** - -> Introduced in GitLab 12.6. - -This file logs information related to the Elasticsearch Integration, including -errors during indexing or searching Elasticsearch. Depending on your installation -method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/elasticsearch.log` -- Installations from source: `/home/git/gitlab/log/elasticsearch.log` - -Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. -Line breaks have been added to the following example line for clarity: - -```json -{ - "severity":"DEBUG", - "time":"2019-10-17T06:23:13.227Z", - "correlation_id":null, - "message":"redacted_search_result", - "class_name":"Milestone", - "id":2, - "ability":"read_milestone", - "current_user_id":2, - "query":"project" -} -``` - -## `exceptions_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17819) in GitLab 12.6. - -This file logs the information about exceptions being tracked by -`Gitlab::ErrorTracking`, which provides a standard and consistent way of -[processing rescued exceptions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/logging.md#exception-handling). -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/exceptions_json.log` -- Installations from source: `/home/git/gitlab/log/exceptions_json.log` - -Each line contains JSON that can be ingested by Elasticsearch. For example: - -```json -{ - "severity": "ERROR", - "time": "2019-12-17T11:49:29.485Z", - "correlation_id": "AbDVUrrTvM1", - "extra.project_id": 55, - "extra.relation_key": "milestones", - "extra.relation_index": 1, - "exception.class": "NoMethodError", - "exception.message": "undefined method `strong_memoize' for #<Gitlab::ImportExport::RelationFactory:0x00007fb5d917c4b0>", - "exception.backtrace": [ - "lib/gitlab/import_export/relation_factory.rb:329:in `unique_relation?'", - "lib/gitlab/import_export/relation_factory.rb:345:in `find_or_create_object!'" - ] -} -``` - -## `service_measurement.log` - -> Introduced in GitLab 13.0. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/service_measurement.log` -- Installations from source: `/home/git/gitlab/log/service_measurement.log` - -It contains only a single structured log with measurements for each service execution. -It contains measurements such as the number of SQL calls, `execution_time`, `gc_stats`, and `memory usage`. - -For example: - -```json -{ "severity":"INFO", "time":"2020-04-22T16:04:50.691Z","correlation_id":"04f1366e-57a1-45b8-88c1-b00b23dc3616","class":"Projects::ImportExport::ExportService","current_user":"John Doe","project_full_path":"group1/test-export","file_path":"/path/to/archive","gc_stats":{"count":{"before":127,"after":127,"diff":0},"heap_allocated_pages":{"before":10369,"after":10369,"diff":0},"heap_sorted_length":{"before":10369,"after":10369,"diff":0},"heap_allocatable_pages":{"before":0,"after":0,"diff":0},"heap_available_slots":{"before":4226409,"after":4226409,"diff":0},"heap_live_slots":{"before":2542709,"after":2641420,"diff":98711},"heap_free_slots":{"before":1683700,"after":1584989,"diff":-98711},"heap_final_slots":{"before":0,"after":0,"diff":0},"heap_marked_slots":{"before":2542704,"after":2542704,"diff":0},"heap_eden_pages":{"before":10369,"after":10369,"diff":0},"heap_tomb_pages":{"before":0,"after":0,"diff":0},"total_allocated_pages":{"before":10369,"after":10369,"diff":0},"total_freed_pages":{"before":0,"after":0,"diff":0},"total_allocated_objects":{"before":24896308,"after":24995019,"diff":98711},"total_freed_objects":{"before":22353599,"after":22353599,"diff":0},"malloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"malloc_increase_bytes_limit":{"before":25804104,"after":25804104,"diff":0},"minor_gc_count":{"before":94,"after":94,"diff":0},"major_gc_count":{"before":33,"after":33,"diff":0},"remembered_wb_unprotected_objects":{"before":34284,"after":34284,"diff":0},"remembered_wb_unprotected_objects_limit":{"before":68568,"after":68568,"diff":0},"old_objects":{"before":2404725,"after":2404725,"diff":0},"old_objects_limit":{"before":4809450,"after":4809450,"diff":0},"oldmalloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"oldmalloc_increase_bytes_limit":{"before":68537556,"after":68537556,"diff":0}},"time_to_finish":0.12298400001600385,"number_of_sql_calls":70,"memory_usage":"0.0 MiB","label":"process_48616"} -``` - -## `geo.log` **(PREMIUM SELF)** - -Geo stores structured log messages in a `geo.log` file. For Omnibus GitLab -installations, this file is at `/var/log/gitlab/gitlab-rails/geo.log`. - -This file contains information about when Geo attempts to sync repositories -and files. Each line in the file contains a separate JSON entry that can be -ingested into (for example, Elasticsearch or Splunk). - -For example: - -```json -{"severity":"INFO","time":"2017-08-06T05:40:16.104Z","message":"Repository update","project_id":1,"source":"repository","resync_repository":true,"resync_wiki":true,"class":"Gitlab::Geo::LogCursor::Daemon","cursor_delay_s":0.038} -``` - -This message shows that Geo detected that a repository update was needed for project `1`. - -## `update_mirror_service_json.log` - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/update_mirror_service_json.log` -- Installations from source: `/home/git/gitlab/log/update_mirror_service_json.log` - -This file contains information about LFS errors that occurred during project mirroring. -While we work to move other project mirroring errors into this log, the [general log](#productionlog) -can be used. - -```json -{ - "severity":"ERROR", - "time":"2020-07-28T23:29:29.473Z", - "correlation_id":"5HgIkCJsO53", - "user_id":"x", - "project_id":"x", - "import_url":"https://mirror-source/group/project.git", - "error_message":"The LFS objects download list couldn't be imported. Error: Unauthorized" -} -``` - -## Registry Logs - -For Omnibus GitLab installations, Container Registry logs are in `/var/log/gitlab/registry/current`. - -## NGINX Logs - -For Omnibus GitLab installations, NGINX logs are in: - -- `/var/log/gitlab/nginx/gitlab_access.log`: A log of requests made to GitLab -- `/var/log/gitlab/nginx/gitlab_error.log`: A log of NGINX errors for GitLab -- `/var/log/gitlab/nginx/gitlab_pages_access.log`: A log of requests made to Pages static sites -- `/var/log/gitlab/nginx/gitlab_pages_error.log`: A log of NGINX errors for Pages static sites -- `/var/log/gitlab/nginx/gitlab_registry_access.log`: A log of requests made to the Container Registry -- `/var/log/gitlab/nginx/gitlab_registry_error.log`: A log of NGINX errors for the Container Registry -- `/var/log/gitlab/nginx/gitlab_mattermost_access.log`: A log of requests made to Mattermost -- `/var/log/gitlab/nginx/gitlab_mattermost_error.log`: A log of NGINX errors for Mattermost - -Below is the default GitLab NGINX access log format: - -```plaintext -$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" -``` - -## Pages Logs - -For Omnibus GitLab installations, Pages logs are in `/var/log/gitlab/gitlab-pages/current`. - -For example: - -```json -{ - "level": "info", - "msg": "GitLab Pages Daemon", - "revision": "52b2899", - "time": "2020-04-22T17:53:12Z", - "version": "1.17.0" -} -{ - "level": "info", - "msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages", - "time": "2020-04-22T17:53:12Z" -} -{ - "gid": 998, - "in-place": false, - "level": "info", - "msg": "running the daemon as unprivileged user", - "time": "2020-04-22T17:53:12Z", - "uid": 998 -} -``` - -## Mattermost Logs - -For Omnibus GitLab installations, Mattermost logs are in these locations: - -- `/var/log/gitlab/mattermost/mattermost.log` -- `/var/log/gitlab/mattermost/current` - -## Workhorse Logs - -For Omnibus GitLab installations, Workhorse logs are in `/var/log/gitlab/gitlab-workhorse/current`. - -## PostgreSQL Logs - -For Omnibus GitLab installations, PostgreSQL logs are in `/var/log/gitlab/postgresql/current`. - -## Prometheus Logs - -For Omnibus GitLab installations, Prometheus logs are in `/var/log/gitlab/prometheus/current`. - -## Redis Logs - -For Omnibus GitLab installations, Redis logs are in `/var/log/gitlab/redis/current`. - -## Alertmanager Logs - -For Omnibus GitLab installations, Alertmanager logs are in `/var/log/gitlab/alertmanager/current`. - -<!-- vale gitlab.Spelling = NO --> - -## Crond Logs - -For Omnibus GitLab installations, crond logs are in `/var/log/gitlab/crond/`. - -<!-- vale gitlab.Spelling = YES --> - -## Grafana Logs - -For Omnibus GitLab installations, Grafana logs are in `/var/log/gitlab/grafana/current`. - -## LogRotate Logs - -For Omnibus GitLab installations, `logrotate` logs are in `/var/log/gitlab/logrotate/current`. - -## GitLab Monitor Logs - -For Omnibus GitLab installations, GitLab Monitor logs are in `/var/log/gitlab/gitlab-monitor/`. - -## GitLab Exporter - -For Omnibus GitLab installations, GitLab Exporter logs are in `/var/log/gitlab/gitlab-exporter/current`. - -## GitLab agent server - -For Omnibus GitLab installations, GitLab agent server logs are -in `/var/log/gitlab/gitlab-kas/current`. - -## Praefect Logs - -For Omnibus GitLab installations, Praefect logs are in `/var/log/gitlab/praefect/`. - -GitLab also tracks [Prometheus metrics for Praefect](gitaly/monitoring.md#monitor-gitaly-cluster). - -## Backup log - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63832) in GitLab 14.1. - -For Omnibus installations, the backup log is located at `/var/log/gitlab/gitlab-rails/backup_json.log`. - -This log is populated when a [GitLab backup is created](../raketasks/backup_restore.md). You can use this log to understand how the backup process performed. - -## Performance bar stats - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48149) in GitLab 13.7. - -Depending on your installation method, this file is located at: - -- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/performance_bar_json.log` -- Installations from source: `/home/git/gitlab/log/performance_bar_json.log` - -Performance bar statistics (currently only duration of SQL queries) are recorded -in that file. For example: - -```json -{"severity":"INFO","time":"2020-12-04T09:29:44.592Z","correlation_id":"33680b1490ccd35981b03639c406a697","filename":"app/models/ci/pipeline.rb","method_path":"app/models/ci/pipeline.rb:each_with_object","request_id":"rYHomD0VJS4","duration_ms":26.889,"count":2,"query_type": "active-record"} -``` - -These statistics are logged on .com only, disabled on self-deployments. - -## Gathering logs - -When [troubleshooting](troubleshooting/index.md) issues that aren't localized to one of the -previously listed components, it's helpful to simultaneously gather multiple logs and statistics -from a GitLab instance. - -NOTE: -GitLab Support often asks for one of these, and maintains the required tools. - -### Briefly tail the main logs - -If the bug or error is readily reproducible, save the main GitLab logs -[to a file](troubleshooting/linux_cheat_sheet.md#files-and-directories) while reproducing the -problem a few times: - -```shell -sudo gitlab-ctl tail | tee /tmp/<case-ID-and-keywords>.log -``` - -Conclude the log gathering with <kbd>Control</kbd> + <kbd>C</kbd>. - -### GitLabSOS - -If performance degradations or cascading errors occur that can't readily be attributed to one -of the previously listed GitLab components, [GitLabSOS](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) -can provide a broader perspective of the GitLab instance. For more details and instructions -to run it, read [the GitLabSOS documentation](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/#gitlabsos). - -### Fast-stats - -[Fast-stats](https://gitlab.com/gitlab-com/support/toolbox/fast-stats) is a tool -for creating and comparing performance statistics from GitLab logs. -For more details and instructions to run it, read the -[documentation for fast-stats](https://gitlab.com/gitlab-com/support/toolbox/fast-stats#usage). +<!-- This redirect file can be deleted after <2022-10-23>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/img/database-query-dialog_v14_3.png b/doc/administration/logs/img/database-query-dialog_v14_3.png Binary files differindex 197cfa17da2..197cfa17da2 100644 --- a/doc/administration/troubleshooting/img/database-query-dialog_v14_3.png +++ b/doc/administration/logs/img/database-query-dialog_v14_3.png diff --git a/doc/administration/troubleshooting/img/network_monitor_xid.png b/doc/administration/logs/img/network_monitor_xid.png Binary files differindex fa64fd1509c..fa64fd1509c 100644 --- a/doc/administration/troubleshooting/img/network_monitor_xid.png +++ b/doc/administration/logs/img/network_monitor_xid.png diff --git a/doc/administration/troubleshooting/img/obtaining-a-session-cookie-for-request_v14_3.png b/doc/administration/logs/img/obtaining-a-session-cookie-for-request_v14_3.png Binary files differindex a63ebf9ecf2..a63ebf9ecf2 100644 --- a/doc/administration/troubleshooting/img/obtaining-a-session-cookie-for-request_v14_3.png +++ b/doc/administration/logs/img/obtaining-a-session-cookie-for-request_v14_3.png diff --git a/doc/administration/troubleshooting/img/paste-request-id-into-progress-bar_v14_3.png b/doc/administration/logs/img/paste-request-id-into-progress-bar_v14_3.png Binary files differindex a19585d7a89..a19585d7a89 100644 --- a/doc/administration/troubleshooting/img/paste-request-id-into-progress-bar_v14_3.png +++ b/doc/administration/logs/img/paste-request-id-into-progress-bar_v14_3.png diff --git a/doc/administration/troubleshooting/img/select-request-id-from-request-selector-drop-down-menu_v14_3.png b/doc/administration/logs/img/select-request-id-from-request-selector-drop-down-menu_v14_3.png Binary files differindex b8314056c9b..b8314056c9b 100644 --- a/doc/administration/troubleshooting/img/select-request-id-from-request-selector-drop-down-menu_v14_3.png +++ b/doc/administration/logs/img/select-request-id-from-request-selector-drop-down-menu_v14_3.png diff --git a/doc/administration/troubleshooting/img/view-pg-details_v14_3.png b/doc/administration/logs/img/view-pg-details_v14_3.png Binary files differindex 1fe12462e4e..1fe12462e4e 100644 --- a/doc/administration/troubleshooting/img/view-pg-details_v14_3.png +++ b/doc/administration/logs/img/view-pg-details_v14_3.png diff --git a/doc/administration/logs/index.md b/doc/administration/logs/index.md new file mode 100644 index 00000000000..d3529de08db --- /dev/null +++ b/doc/administration/logs/index.md @@ -0,0 +1,1182 @@ +--- +stage: Monitor +group: Respond +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Log system **(FREE SELF)** + +GitLab has an advanced log system where everything is logged, so you +can analyze your instance using various system log files. In addition to +system log files, GitLab Enterprise Edition provides [Audit Events](../audit_events.md). + +System log files are typically plain text in a standard log file format. +This guide talks about how to read and use these system log files. + +Read more about the log system and using the logs: + +- [Customize logging on Omnibus GitLab installations](https://docs.gitlab.com/omnibus/settings/logs.html) +including adjusting log retention, log forwarding, +switching logs from JSON to plain text logging, and more. +- [How to parse and analyze JSON logs](../logs/log_parsing.md). + +## Log Levels + +Each log message has an assigned log level that indicates its importance and verbosity. +Each logger has an assigned minimum log level. +A logger emits a log message only if its log level is equal to or above the minimum log level. + +The following log levels are supported: + +| Level | Name | +|:------|:----------| +| 0 | `DEBUG` | +| 1 | `INFO` | +| 2 | `WARN` | +| 3 | `ERROR` | +| 4 | `FATAL` | +| 5 | `UNKNOWN` | + +GitLab loggers emit all log messages because they are set to `DEBUG` by default. + +### Override default log level + +You can override the minimum log level for GitLab loggers using the `GITLAB_LOG_LEVEL` environment variable. +Valid values are either a value of `0` to `5`, or the name of the log level. + +Example: + +```shell +GITLAB_LOG_LEVEL=info +``` + +For some services, other log levels are in place that are not affected by this setting. +Some of these services have their own environment variables to override the log level. For example: + +| Service | Log level | Environment variable | +|:---------------------|:----------|:---------------------| +| GitLab API | `INFO` | | +| GitLab Cleanup | `INFO` | `DEBUG` | +| GitLab Doctor | `INFO` | `VERBOSE` | +| GitLab Export | `INFO` | `EXPORT_DEBUG` | +| GitLab Geo | `INFO` | | +| GitLab Import | `INFO` | `IMPORT_DEBUG` | +| GitLab QA Runtime | `INFO` | `QA_LOG_LEVEL` | +| Google APIs | `INFO` | | +| Rack Timeout | `ERROR` | | +| Sidekiq (server) | `INFO` | | +| Snowplow Tracker | `FATAL` | | +| gRPC Client (Gitaly) | `WARN` | `GRPC_LOG_LEVEL` | + +## Log Rotation + +The logs for a given service may be managed and rotated by: + +- `logrotate` +- `svlogd` (`runit`'s service logging daemon) +- `logrotate` and `svlogd` +- Or not at all + +The following table includes information about what's responsible for managing and rotating logs for +the included services. Logs +[managed by `svlogd`](https://docs.gitlab.com/omnibus/settings/logs.html#runit-logs) +are written to a file called `current`. The `logrotate` service built into GitLab +[manages all logs](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate) +except those captured by `runit`. + +| Log type | Managed by logrotate | Managed by svlogd/runit | +|:------------------------------------------------|:------------------------|:------------------------| +| [Alertmanager Logs](#alertmanager-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Crond Logs](#crond-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Gitaly](#gitaly-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [GitLab Exporter for Omnibus](#gitlab-exporter) | **{dotted-circle}** No | **{check-circle}** Yes | +| [GitLab Pages Logs](#pages-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| GitLab Rails | **{check-circle}** Yes | **{dotted-circle}** No | +| [GitLab Shell Logs](#gitlab-shelllog) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Grafana Logs](#grafana-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [LogRotate Logs](#logrotate-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Mailroom](#mail_room_jsonlog-default) | **{check-circle}** Yes | **{check-circle}** Yes | +| [NGINX](#nginx-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [PostgreSQL Logs](#postgresql-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Praefect Logs](#praefect-logs) | **{dotted-circle}** Yes | **{check-circle}** Yes | +| [Prometheus Logs](#prometheus-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Puma](#puma-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [Redis Logs](#redis-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Registry Logs](#registry-logs) | **{dotted-circle}** No | **{check-circle}** Yes | +| [Workhorse Logs](#workhorse-logs) | **{check-circle}** Yes | **{check-circle}** Yes | + +## `production_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/production_json.log` +- Installations from source: `/home/git/gitlab/log/production_json.log` + +It contains a structured log for Rails controller requests received from +GitLab, thanks to [Lograge](https://github.com/roidrage/lograge/). +Requests from the API are logged to a separate file in `api_json.log`. + +Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. +Line breaks were added to examples for legibility: + +```json +{ + "method":"GET", + "path":"/gitlab/gitlab-foss/issues/1234", + "format":"html", + "controller":"Projects::IssuesController", + "action":"show", + "status":200, + "time":"2017-08-08T20:15:54.821Z", + "params":[{"key":"param_key","value":"param_value"}], + "remote_ip":"18.245.0.1", + "user_id":1, + "username":"admin", + "queue_duration_s":0.0, + "gitaly_calls":16, + "gitaly_duration_s":0.16, + "redis_calls":115, + "redis_duration_s":0.13, + "redis_read_bytes":1507378, + "redis_write_bytes":2920, + "correlation_id":"O1SdybnnIq7", + "cpu_s":17.50, + "db_duration_s":0.08, + "view_duration_s":2.39, + "duration_s":20.54, + "pid": 81836, + "worker_id":"puma_0" +} +``` + +This example was a GET request for a specific +issue. Each line also contains performance data, with times in +seconds: + +- `duration_s`: Total time to retrieve the request +- `queue_duration_s`: Total time the request was queued inside GitLab Workhorse +- `view_duration_s`: Total time inside the Rails views +- `db_duration_s`: Total time to retrieve data from PostgreSQL +- `cpu_s`: Total time spent on CPU +- `gitaly_duration_s`: Total time by Gitaly calls +- `gitaly_calls`: Total number of calls made to Gitaly +- `redis_calls`: Total number of calls made to Redis +- `redis_duration_s`: Total time to retrieve data from Redis +- `redis_read_bytes`: Total bytes read from Redis +- `redis_write_bytes`: Total bytes written to Redis +- `redis_<instance>_calls`: Total number of calls made to a Redis instance +- `redis_<instance>_duration_s`: Total time to retrieve data from a Redis instance +- `redis_<instance>_read_bytes`: Total bytes read from a Redis instance +- `redis_<instance>_write_bytes`: Total bytes written to a Redis instance +- `pid`: The worker's Linux process ID (changes when workers restart) +- `worker_id`: The worker's logical ID (does not change when workers restart) + +User clone and fetch activity using HTTP transport appears in the log as `action: git_upload_pack`. + +In addition, the log contains the originating IP address, +(`remote_ip`), the user's ID (`user_id`), and username (`username`). + +Some endpoints (such as `/search`) may make requests to Elasticsearch if using +[Advanced Search](../../user/search/advanced_search.md). These +additionally log `elasticsearch_calls` and `elasticsearch_call_duration_s`, +which correspond to: + +- `elasticsearch_calls`: Total number of calls to Elasticsearch +- `elasticsearch_duration_s`: Total time taken by Elasticsearch calls +- `elasticsearch_timed_out_count`: Total number of calls to Elasticsearch that + timed out and therefore returned partial results + +ActionCable connection and subscription events are also logged to this file and they follow the +previous format. The `method`, `path`, and `format` fields are not applicable, and are always empty. +The ActionCable connection or channel class is used as the `controller`. + +```json +{ + "method":null, + "path":null, + "format":null, + "controller":"IssuesChannel", + "action":"subscribe", + "status":200, + "time":"2020-05-14T19:46:22.008Z", + "params":[{"key":"project_path","value":"gitlab/gitlab-foss"},{"key":"iid","value":"1"}], + "remote_ip":"127.0.0.1", + "user_id":1, + "username":"admin", + "ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:76.0) Gecko/20100101 Firefox/76.0", + "correlation_id":"jSOIEynHCUa", + "duration_s":0.32566 +} +``` + +NOTE: +Starting with GitLab 12.5, if an error occurs, an +`exception` field is included with `class`, `message`, and +`backtrace`. Previous versions included an `error` field instead of +`exception.class` and `exception.message`. For example: + +```json +{ + "method": "GET", + "path": "/admin", + "format": "html", + "controller": "Admin::DashboardController", + "action": "index", + "status": 500, + "time": "2019-11-14T13:12:46.156Z", + "params": [], + "remote_ip": "127.0.0.1", + "user_id": 1, + "username": "root", + "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:70.0) Gecko/20100101 Firefox/70.0", + "queue_duration": 274.35, + "correlation_id": "KjDVUhNvvV3", + "queue_duration_s":0.0, + "gitaly_calls":16, + "gitaly_duration_s":0.16, + "redis_calls":115, + "redis_duration_s":0.13, + "correlation_id":"O1SdybnnIq7", + "cpu_s":17.50, + "db_duration_s":0.08, + "view_duration_s":2.39, + "duration_s":20.54, + "pid": 81836, + "worker_id": "puma_0", + "exception.class": "NameError", + "exception.message": "undefined local variable or method `adsf' for #<Admin::DashboardController:0x00007ff3c9648588>", + "exception.backtrace": [ + "app/controllers/admin/dashboard_controller.rb:11:in `index'", + "ee/app/controllers/ee/admin/dashboard_controller.rb:14:in `index'", + "ee/lib/gitlab/ip_address_state.rb:10:in `with'", + "ee/app/controllers/ee/application_controller.rb:43:in `set_current_ip_address'", + "lib/gitlab/session.rb:11:in `with_session'", + "app/controllers/application_controller.rb:450:in `set_session_storage'", + "app/controllers/application_controller.rb:444:in `set_locale'", + "ee/lib/gitlab/jira/middleware.rb:19:in `call'" + ] +} +``` + +## `production.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/production.log` +- Installations from source: `/home/git/gitlab/log/production.log` + +It contains information about all performed requests. You can see the +URL and type of request, IP address, and what parts of code were +involved to service this particular request. Also, you can see all SQL +requests performed, and how much time each took. This task is +more useful for GitLab contributors and developers. Use part of this log +file when you're reporting bugs. For example: + +```plaintext +Started GET "/gitlabhq/yaml_db/tree/master" for 168.111.56.1 at 2015-02-12 19:34:53 +0200 +Processing by Projects::TreeController#show as HTML + Parameters: {"project_id"=>"gitlabhq/yaml_db", "id"=>"master"} + + ... [CUT OUT] + + Namespaces"."created_at" DESC, "namespaces"."id" DESC LIMIT 1 [["id", 26]] + CACHE (0.0ms) SELECT "members".* FROM "members" WHERE "members"."source_type" = 'Project' AND "members"."type" IN ('ProjectMember') AND "members"."source_id" = $1 AND "members"."source_type" = $2 AND "members"."user_id" = 1 ORDER BY "members"."created_at" DESC, "members"."id" DESC LIMIT 1 [["source_id", 18], ["source_type", "Project"]] + CACHE (0.0ms) SELECT "members".* FROM "members" WHERE "members"."source_type" = 'Project' AND "members". + (1.4ms) SELECT COUNT(*) FROM "merge_requests" WHERE "merge_requests"."target_project_id" = $1 AND ("merge_requests"."state" IN ('opened','reopened')) [["target_project_id", 18]] + Rendered layouts/nav/_project.html.haml (28.0ms) + Rendered layouts/_collapse_button.html.haml (0.2ms) + Rendered layouts/_flash.html.haml (0.1ms) + Rendered layouts/_page.html.haml (32.9ms) +Completed 200 OK in 166ms (Views: 117.4ms | ActiveRecord: 27.2ms) +``` + +In this example, the server processed an HTTP request with URL +`/gitlabhq/yaml_db/tree/master` from IP `168.111.56.1` at `2015-02-12 19:34:53 +0200`. +The request was processed by `Projects::TreeController`. + +## `api_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/api_json.log` +- Installations from source: `/home/git/gitlab/log/api_json.log` + +It helps you see requests made directly to the API. For example: + +```json +{ + "time":"2018-10-29T12:49:42.123Z", + "severity":"INFO", + "duration":709.08, + "db":14.59, + "view":694.49, + "status":200, + "method":"GET", + "path":"/api/v4/projects", + "params":[{"key":"action","value":"git-upload-pack"},{"key":"changes","value":"_any"},{"key":"key_id","value":"secret"},{"key":"secret_token","value":"[FILTERED]"}], + "host":"localhost", + "remote_ip":"::1", + "ua":"Ruby", + "route":"/api/:version/projects", + "user_id":1, + "username":"root", + "queue_duration":100.31, + "gitaly_calls":30, + "gitaly_duration":5.36, + "pid": 81836, + "worker_id": "puma_0", + ... +} +``` + +This entry shows an internal endpoint accessed to check whether an +associated SSH key can download the project in question by using a `git fetch` or +`git clone`. In this example, we see: + +- `duration`: Total time in milliseconds to retrieve the request +- `queue_duration`: Total time in milliseconds the request was queued inside GitLab Workhorse +- `method`: The HTTP method used to make the request +- `path`: The relative path of the query +- `params`: Key-value pairs passed in a query string or HTTP body (sensitive parameters, such as passwords and tokens, are filtered out) +- `ua`: The User-Agent of the requester + +## `application.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application.log` +- Installations from source: `/home/git/gitlab/log/application.log` + +It helps you discover events happening in your instance such as user creation +and project deletion. For example: + +```plaintext +October 06, 2014 11:56: User "Administrator" (admin@example.com) was created +October 06, 2014 11:56: Documentcloud created a new project "Documentcloud / Underscore" +October 06, 2014 11:56: Gitlab Org created a new project "Gitlab Org / Gitlab Ce" +October 07, 2014 11:25: User "Claudie Hodkiewicz" (nasir_stehr@olson.co.uk) was removed +October 07, 2014 11:25: Project "project133" was removed +``` + +## `application_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22812) in GitLab 12.7. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application_json.log` +- Installations from source: `/home/git/gitlab/log/application_json.log` + +It contains the JSON version of the logs in `application.log`, like this example: + +```json +{ + "severity":"INFO", + "time":"2020-01-14T13:35:15.466Z", + "correlation_id":"3823a1550b64417f9c9ed8ee0f48087e", + "message":"User \"Administrator\" (admin@example.com) was created" +} +{ + "severity":"INFO", + "time":"2020-01-14T13:35:15.466Z", + "correlation_id":"78e3df10c9a18745243d524540bd5be4", + "message":"Project \"project133\" was removed" +} +``` + +## `integrations_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/integrations_json.log` +- Installations from source: `/home/git/gitlab/log/integrations_json.log` + +It contains information about [integration](../../user/project/integrations/index.md) +activities, such as Jira, Asana, and irker services. It uses JSON format, +like this example: + +```json +{ + "severity":"ERROR", + "time":"2018-09-06T14:56:20.439Z", + "service_class":"Integrations::Jira", + "project_id":8, + "project_path":"h5bp/html5-boilerplate", + "message":"Error sending message", + "client_url":"http://jira.gitlap.com:8080", + "error":"execution expired" +} +{ + "severity":"INFO", + "time":"2018-09-06T17:15:16.365Z", + "service_class":"Integrations::Jira", + "project_id":3, + "project_path":"namespace2/project2", + "message":"Successfully posted", + "client_url":"http://jira.example.com" +} +``` + +## `kubernetes.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/kubernetes.log` +- Installations from source: `/home/git/gitlab/log/kubernetes.log` + +It logs information related to the Kubernetes Integration, including errors +during installing cluster applications on your managed Kubernetes +clusters. + +Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. +Line breaks have been added to the following example for clarity: + +```json +{ + "severity":"ERROR", + "time":"2018-11-23T15:14:54.652Z", + "exception":"Kubeclient::HttpError", + "error_code":401, + "service":"Clusters::Applications::CheckInstallationProgressService", + "app_id":14, + "project_ids":[1], + "group_ids":[], + "message":"Unauthorized" +} +{ + "severity":"ERROR", + "time":"2018-11-23T15:42:11.647Z", + "exception":"Kubeclient::HttpError", + "error_code":null, + "service":"Clusters::Applications::InstallService", + "app_id":2, + "project_ids":[19], + "group_ids":[], + "message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)" +} +``` + +## `git_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/git_json.log` +- Installations from source: `/home/git/gitlab/log/git_json.log` + +After GitLab version 12.2, this file was renamed from `githost.log` to +`git_json.log` and stored in JSON format. + +GitLab has to interact with Git repositories, but in some rare cases +something can go wrong. If this happens, you need to know exactly what +happened. This log file contains all failed requests from GitLab to Git +repositories. In the majority of cases this file is useful for developers +only. For example: + +```json +{ + "severity":"ERROR", + "time":"2019-07-19T22:16:12.528Z", + "correlation_id":"FeGxww5Hj64", + "message":"Command failed [1]: /usr/bin/git --git-dir=/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/group184/gitlabhq/.git --work-tree=/Users/vsizov/gitlab-development-kit/gitlab/tmp/tests/gitlab-satellites/group184/gitlabhq merge --no-ff -mMerge branch 'feature_conflict' into 'feature' source/feature_conflict\n\nerror: failed to push some refs to '/Users/vsizov/gitlab-development-kit/repositories/gitlabhq/gitlab_git.git'" +} +``` + +## `audit_json.log` **(FREE)** + +NOTE: +GitLab Free tracks a small number of different audit events. +GitLab Premium tracks many more. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/audit_json.log` +- Installations from source: `/home/git/gitlab/log/audit_json.log` + +Changes to group or project settings and memberships (`target_details`) +are logged to this file. For example: + +```json +{ + "severity":"INFO", + "time":"2018-10-17T17:38:22.523Z", + "author_id":3, + "entity_id":2, + "entity_type":"Project", + "change":"visibility", + "from":"Private", + "to":"Public", + "author_name":"John Doe4", + "target_id":2, + "target_type":"Project", + "target_details":"namespace2/project2" +} +``` + +## Sidekiq Logs + +NOTE: +In Omnibus GitLab `12.10` or earlier, the Sidekiq log is at `/var/log/gitlab/gitlab-rails/sidekiq.log`. + +For Omnibus GitLab installations, some Sidekiq logs are in `/var/log/gitlab/sidekiq/current` +and as follows. + +### `sidekiq.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/sidekiq/current` +- Installations from source: `/home/git/gitlab/log/sidekiq.log` + +GitLab uses background jobs for processing tasks which can take a long +time. All information about processing these jobs are written down to +this file. For example: + +```plaintext +2014-06-10T07:55:20Z 2037 TID-tm504 ERROR: /opt/bitnami/apps/discourse/htdocs/vendor/bundle/ruby/1.9.1/gems/redis-3.0.7/lib/redis/client.rb:228:in `read' +2014-06-10T18:18:26Z 14299 TID-55uqo INFO: Booting Sidekiq 3.0.0 with redis options {:url=>"redis://localhost:6379/0", :namespace=>"sidekiq"} +``` + +Instead of the previous format, you can opt to generate JSON logs for +Sidekiq. For example: + +```json +{ + "severity":"INFO", + "time":"2018-04-03T22:57:22.071Z", + "queue":"cronjob:update_all_mirrors", + "args":[], + "class":"UpdateAllMirrorsWorker", + "retry":false, + "queue_namespace":"cronjob", + "jid":"06aeaa3b0aadacf9981f368e", + "created_at":"2018-04-03T22:57:21.930Z", + "enqueued_at":"2018-04-03T22:57:21.931Z", + "pid":10077, + "worker_id":"sidekiq_0", + "message":"UpdateAllMirrorsWorker JID-06aeaa3b0aadacf9981f368e: done: 0.139 sec", + "job_status":"done", + "duration":0.139, + "completed_at":"2018-04-03T22:57:22.071Z", + "db_duration":0.05, + "db_duration_s":0.0005, + "gitaly_duration":0, + "gitaly_calls":0 +} +``` + +For Omnibus GitLab installations, add the configuration option: + +```ruby +sidekiq['log_format'] = 'json' +``` + +For installations from source, edit the `gitlab.yml` and set the Sidekiq +`log_format` configuration option: + +```yaml + ## Sidekiq + sidekiq: + log_format: json +``` + +### `sidekiq_client.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26586) in GitLab 12.9. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/sidekiq_client.log` +- Installations from source: `/home/git/gitlab/log/sidekiq_client.log` + +This file contains logging information about jobs before Sidekiq starts +processing them, such as before being enqueued. + +This log file follows the same structure as +[`sidekiq.log`](#sidekiqlog), so it is structured as JSON if +you've configured this for Sidekiq as mentioned above. + +## `gitlab-shell.log` + +GitLab Shell is used by GitLab for executing Git commands and provide SSH +access to Git repositories. + +### For GitLab versions 12.10 and up + +Information containing `git-{upload-pack,receive-pack}` requests is at +`/var/log/gitlab/gitlab-shell/gitlab-shell.log`. Information about hooks to +GitLab Shell from Gitaly is at `/var/log/gitlab/gitaly/current`. + +Example log entries for `/var/log/gitlab/gitlab-shell/gitlab-shell.log`: + +```json +{ + "duration_ms": 74.104, + "level": "info", + "method": "POST", + "msg": "Finished HTTP request", + "time": "2020-04-17T20:28:46Z", + "url": "http://127.0.0.1:8080/api/v4/internal/allowed" +} +{ + "command": "git-upload-pack", + "git_protocol": "", + "gl_project_path": "root/example", + "gl_repository": "project-1", + "level": "info", + "msg": "executing git command", + "time": "2020-04-17T20:28:46Z", + "user_id": "user-1", + "username": "root" +} +``` + +Example log entries for `/var/log/gitlab/gitaly/current`: + +```json +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/allowed", + "duration": 0.058012959, + "gitaly_embedded": true, + "pid": 16636, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T20:29:08+00:00" +} +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/pre_receive", + "duration": 0.031022552, + "gitaly_embedded": true, + "pid": 16636, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T20:29:08+00:00" +} +``` + +### For GitLab versions 12.5 through 12.9 + +For GitLab 12.5 to 12.9, depending on your installation method, this +file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitaly/gitlab-shell.log` +- Installation from source: `/home/git/gitaly/gitlab-shell.log` + +Example log entries: + +```json +{ + "method": "POST", + "url": "http://127.0.0.1:8080/api/v4/internal/post_receive", + "duration": 0.031809164, + "gitaly_embedded": true, + "pid": 27056, + "level": "info", + "msg": "finished HTTP request", + "time": "2020-04-17T16:24:38+00:00" +} +``` + +### For GitLab 12.5 and earlier + +For GitLab 12.5 and earlier, the file is at `/var/log/gitlab/gitlab-shell/gitlab-shell.log`. + +Example log entries: + +```plaintext +I, [2015-02-13T06:17:00.671315 #9291] INFO -- : Adding project root/example.git at </var/opt/gitlab/git-data/repositories/root/dcdcdcdcd.git>. +I, [2015-02-13T06:17:00.679433 #9291] INFO -- : Moving existing hooks directory and symlinking global hooks directory for /var/opt/gitlab/git-data/repositories/root/example.git. +``` + +User clone/fetch activity using SSH transport appears in this log as +`executing git command <gitaly-upload-pack...`. + +## Gitaly Logs + +This file is in `/var/log/gitlab/gitaly/current` and is produced by [runit](http://smarden.org/runit/). +`runit` is packaged with Omnibus GitLab and a brief explanation of its purpose +is available [in the Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/architecture/#runit). +[Log files are rotated](http://smarden.org/runit/svlogd.8.html), renamed in +Unix timestamp format, and `gzip`-compressed (like `@1584057562.s`). + +### `grpc.log` + +This file is at `/var/log/gitlab/gitlab-rails/grpc.log` for Omnibus GitLab +packages. Native [gRPC](https://grpc.io/) logging used by Gitaly. + +### `gitaly_ruby_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2678) in GitLab 13.6. + +This file is at `/var/log/gitlab/gitaly/gitaly_ruby_json.log` and is +produced by [`gitaly-ruby`](../gitaly/reference.md#gitaly-ruby). It contains an +access log of gRPC calls made by Gitaly to `gitaly-ruby`. + +### `gitaly_hooks.log` + +This file is at `/var/log/gitlab/gitaly/gitaly_hooks.log` and is +produced by `gitaly-hooks` command. It also contains records about +failures received during processing of the responses from GitLab API. + +## Puma Logs + +### `puma_stdout.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/puma/puma_stdout.log` +- Installations from source: `/home/git/gitlab/log/puma_stdout.log` + +### `puma_stderr.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/puma/puma_stderr.log` +- Installations from source: `/home/git/gitlab/log/puma_stderr.log` + +## `repocheck.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/repocheck.log` +- Installations from source: `/home/git/gitlab/log/repocheck.log` + +It logs information whenever a [repository check is run](../repository_checks.md) +on a project. + +## `importer.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/importer.log` +- Installations from source: `/home/git/gitlab/log/importer.log` + +It logs the progress of the import process. + +## `exporter.log` + +> Introduced in GitLab 13.1. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/exporter.log` +- Installations from source: `/home/git/gitlab/log/exporter.log` + +It logs the progress of the export process. + +## `features_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 13.7. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/features_json.log` +- Installations from source: `/home/git/gitlab/log/features_json.log` + +The modification events from [Feature flags in development of GitLab](../../development/feature_flags/index.md) +are recorded in this file. For example: + +```json +{"severity":"INFO","time":"2020-11-24T02:30:59.860Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"true"} +{"severity":"INFO","time":"2020-11-24T02:31:29.108Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"true"} +{"severity":"INFO","time":"2020-11-24T02:31:29.129Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable","extra.thing":"false"} +{"severity":"INFO","time":"2020-11-24T02:31:29.177Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable","extra.thing":"Project:1"} +{"severity":"INFO","time":"2020-11-24T02:31:29.183Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable","extra.thing":"Project:1"} +{"severity":"INFO","time":"2020-11-24T02:31:29.188Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable_percentage_of_time","extra.percentage":"50"} +{"severity":"INFO","time":"2020-11-24T02:31:29.193Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable_percentage_of_time"} +{"severity":"INFO","time":"2020-11-24T02:31:29.198Z","correlation_id":null,"key":"cd_auto_rollback","action":"enable_percentage_of_actors","extra.percentage":"50"} +{"severity":"INFO","time":"2020-11-24T02:31:29.203Z","correlation_id":null,"key":"cd_auto_rollback","action":"disable_percentage_of_actors"} +{"severity":"INFO","time":"2020-11-24T02:31:29.329Z","correlation_id":null,"key":"cd_auto_rollback","action":"remove"} +``` + +## `auth.log` + +> Introduced in GitLab 12.0. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/auth.log` +- Installations from source: `/home/git/gitlab/log/auth.log` + +This log records: + +- Requests over the [Rate Limit](../../user/admin_area/settings/rate_limits_on_raw_endpoints.md) on raw endpoints. +- [Protected paths](../../user/admin_area/settings/protected_paths.md) abusive requests. +- In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and later, + user ID and username, if available. + +## `graphql_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 12.0. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/graphql_json.log` +- Installations from source: `/home/git/gitlab/log/graphql_json.log` + +GraphQL queries are recorded in the file. For example: + +```json +{"query_string":"query IntrospectionQuery{__schema {queryType { name },mutationType { name }}}...(etc)","variables":{"a":1,"b":2},"complexity":181,"depth":1,"duration_s":7} +``` + +## `migrations.log` + +> Introduced in GitLab 12.3. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/migrations.log` +- Installations from source: `/home/git/gitlab/log/migrations.log` + +## `mail_room_json.log` (default) + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186) in GitLab 12.6. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/mailroom/current` +- Installations from source: `/home/git/gitlab/log/mail_room_json.log` + +This structured log file records internal activity in the `mail_room` gem. +Its name and path are configurable, so the name and path may not match the above. + +## Reconfigure logs + +Reconfigure log files are in `/var/log/gitlab/reconfigure` for Omnibus GitLab +packages. Installations from source don't have reconfigure logs. A reconfigure log +is populated whenever `gitlab-ctl reconfigure` is run manually or as part of an upgrade. + +Reconfigure logs files are named according to the UNIX timestamp of when the reconfigure +was initiated, such as `1509705644.log` + +## `sidekiq_exporter.log` and `web_exporter.log` + +If Prometheus metrics and the Sidekiq Exporter are both enabled, Sidekiq +starts a Web server and listen to the defined port (default: +`8082`). By default, Sidekiq Exporter access logs are disabled but can +be enabled based on your installation method: + +- Omnibus GitLab: Use the `sidekiq['exporter_log_enabled'] = true` + option in `/etc/gitlab/gitlab.rb` +- Installations from source: Use the `sidekiq_exporter.log_enabled` option + in `gitlab.yml` + +When enabled, depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/sidekiq_exporter.log` +- Installations from source: `/home/git/gitlab/log/sidekiq_exporter.log` + +If Prometheus metrics and the Web Exporter are both enabled, Puma +starts a Web server and listen to the defined port (default: `8083`), and access logs +are generated in a location based on your installation method: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/web_exporter.log` +- Installations from source: `/home/git/gitlab/log/web_exporter.log` + +## `database_load_balancing.log` **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/15442) in GitLab 12.3. + +Contains details of GitLab [Database Load Balancing](../postgresql/database_load_balancing.md). +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/database_load_balancing.log` +- Installations from source: `/home/git/gitlab/log/database_load_balancing.log` + +## `elasticsearch.log` **(PREMIUM SELF)** + +> Introduced in GitLab 12.6. + +This file logs information related to the Elasticsearch Integration, including +errors during indexing or searching Elasticsearch. Depending on your installation +method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/elasticsearch.log` +- Installations from source: `/home/git/gitlab/log/elasticsearch.log` + +Each line contains JSON that can be ingested by services like Elasticsearch and Splunk. +Line breaks have been added to the following example line for clarity: + +```json +{ + "severity":"DEBUG", + "time":"2019-10-17T06:23:13.227Z", + "correlation_id":null, + "message":"redacted_search_result", + "class_name":"Milestone", + "id":2, + "ability":"read_milestone", + "current_user_id":2, + "query":"project" +} +``` + +## `exceptions_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17819) in GitLab 12.6. + +This file logs the information about exceptions being tracked by +`Gitlab::ErrorTracking`, which provides a standard and consistent way of +[processing rescued exceptions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/logging.md#exception-handling). +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/exceptions_json.log` +- Installations from source: `/home/git/gitlab/log/exceptions_json.log` + +Each line contains JSON that can be ingested by Elasticsearch. For example: + +```json +{ + "severity": "ERROR", + "time": "2019-12-17T11:49:29.485Z", + "correlation_id": "AbDVUrrTvM1", + "extra.project_id": 55, + "extra.relation_key": "milestones", + "extra.relation_index": 1, + "exception.class": "NoMethodError", + "exception.message": "undefined method `strong_memoize' for #<Gitlab::ImportExport::RelationFactory:0x00007fb5d917c4b0>", + "exception.backtrace": [ + "lib/gitlab/import_export/relation_factory.rb:329:in `unique_relation?'", + "lib/gitlab/import_export/relation_factory.rb:345:in `find_or_create_object!'" + ] +} +``` + +## `service_measurement.log` + +> Introduced in GitLab 13.0. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/service_measurement.log` +- Installations from source: `/home/git/gitlab/log/service_measurement.log` + +It contains only a single structured log with measurements for each service execution. +It contains measurements such as the number of SQL calls, `execution_time`, `gc_stats`, and `memory usage`. + +For example: + +```json +{ "severity":"INFO", "time":"2020-04-22T16:04:50.691Z","correlation_id":"04f1366e-57a1-45b8-88c1-b00b23dc3616","class":"Projects::ImportExport::ExportService","current_user":"John Doe","project_full_path":"group1/test-export","file_path":"/path/to/archive","gc_stats":{"count":{"before":127,"after":127,"diff":0},"heap_allocated_pages":{"before":10369,"after":10369,"diff":0},"heap_sorted_length":{"before":10369,"after":10369,"diff":0},"heap_allocatable_pages":{"before":0,"after":0,"diff":0},"heap_available_slots":{"before":4226409,"after":4226409,"diff":0},"heap_live_slots":{"before":2542709,"after":2641420,"diff":98711},"heap_free_slots":{"before":1683700,"after":1584989,"diff":-98711},"heap_final_slots":{"before":0,"after":0,"diff":0},"heap_marked_slots":{"before":2542704,"after":2542704,"diff":0},"heap_eden_pages":{"before":10369,"after":10369,"diff":0},"heap_tomb_pages":{"before":0,"after":0,"diff":0},"total_allocated_pages":{"before":10369,"after":10369,"diff":0},"total_freed_pages":{"before":0,"after":0,"diff":0},"total_allocated_objects":{"before":24896308,"after":24995019,"diff":98711},"total_freed_objects":{"before":22353599,"after":22353599,"diff":0},"malloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"malloc_increase_bytes_limit":{"before":25804104,"after":25804104,"diff":0},"minor_gc_count":{"before":94,"after":94,"diff":0},"major_gc_count":{"before":33,"after":33,"diff":0},"remembered_wb_unprotected_objects":{"before":34284,"after":34284,"diff":0},"remembered_wb_unprotected_objects_limit":{"before":68568,"after":68568,"diff":0},"old_objects":{"before":2404725,"after":2404725,"diff":0},"old_objects_limit":{"before":4809450,"after":4809450,"diff":0},"oldmalloc_increase_bytes":{"before":140032,"after":6650240,"diff":6510208},"oldmalloc_increase_bytes_limit":{"before":68537556,"after":68537556,"diff":0}},"time_to_finish":0.12298400001600385,"number_of_sql_calls":70,"memory_usage":"0.0 MiB","label":"process_48616"} +``` + +## `geo.log` **(PREMIUM SELF)** + +Geo stores structured log messages in a `geo.log` file. For Omnibus GitLab +installations, this file is at `/var/log/gitlab/gitlab-rails/geo.log`. + +This file contains information about when Geo attempts to sync repositories +and files. Each line in the file contains a separate JSON entry that can be +ingested into (for example, Elasticsearch or Splunk). + +For example: + +```json +{"severity":"INFO","time":"2017-08-06T05:40:16.104Z","message":"Repository update","project_id":1,"source":"repository","resync_repository":true,"resync_wiki":true,"class":"Gitlab::Geo::LogCursor::Daemon","cursor_delay_s":0.038} +``` + +This message shows that Geo detected that a repository update was needed for project `1`. + +## `update_mirror_service_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/update_mirror_service_json.log` +- Installations from source: `/home/git/gitlab/log/update_mirror_service_json.log` + +This file contains information about LFS errors that occurred during project mirroring. +While we work to move other project mirroring errors into this log, the [general log](#productionlog) +can be used. + +```json +{ + "severity":"ERROR", + "time":"2020-07-28T23:29:29.473Z", + "correlation_id":"5HgIkCJsO53", + "user_id":"x", + "project_id":"x", + "import_url":"https://mirror-source/group/project.git", + "error_message":"The LFS objects download list couldn't be imported. Error: Unauthorized" +} +``` + +## Registry Logs + +For Omnibus GitLab installations, Container Registry logs are in `/var/log/gitlab/registry/current`. + +## NGINX Logs + +For Omnibus GitLab installations, NGINX logs are in: + +- `/var/log/gitlab/nginx/gitlab_access.log`: A log of requests made to GitLab +- `/var/log/gitlab/nginx/gitlab_error.log`: A log of NGINX errors for GitLab +- `/var/log/gitlab/nginx/gitlab_pages_access.log`: A log of requests made to Pages static sites +- `/var/log/gitlab/nginx/gitlab_pages_error.log`: A log of NGINX errors for Pages static sites +- `/var/log/gitlab/nginx/gitlab_registry_access.log`: A log of requests made to the Container Registry +- `/var/log/gitlab/nginx/gitlab_registry_error.log`: A log of NGINX errors for the Container Registry +- `/var/log/gitlab/nginx/gitlab_mattermost_access.log`: A log of requests made to Mattermost +- `/var/log/gitlab/nginx/gitlab_mattermost_error.log`: A log of NGINX errors for Mattermost + +Below is the default GitLab NGINX access log format: + +```plaintext +$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" +``` + +## Pages Logs + +For Omnibus GitLab installations, Pages logs are in `/var/log/gitlab/gitlab-pages/current`. + +For example: + +```json +{ + "level": "info", + "msg": "GitLab Pages Daemon", + "revision": "52b2899", + "time": "2020-04-22T17:53:12Z", + "version": "1.17.0" +} +{ + "level": "info", + "msg": "URL: https://gitlab.com/gitlab-org/gitlab-pages", + "time": "2020-04-22T17:53:12Z" +} +{ + "gid": 998, + "in-place": false, + "level": "info", + "msg": "running the daemon as unprivileged user", + "time": "2020-04-22T17:53:12Z", + "uid": 998 +} +``` + +## Mattermost Logs + +For Omnibus GitLab installations, Mattermost logs are in these locations: + +- `/var/log/gitlab/mattermost/mattermost.log` +- `/var/log/gitlab/mattermost/current` + +## Workhorse Logs + +For Omnibus GitLab installations, Workhorse logs are in `/var/log/gitlab/gitlab-workhorse/current`. + +## PostgreSQL Logs + +For Omnibus GitLab installations, PostgreSQL logs are in `/var/log/gitlab/postgresql/current`. + +## Prometheus Logs + +For Omnibus GitLab installations, Prometheus logs are in `/var/log/gitlab/prometheus/current`. + +## Redis Logs + +For Omnibus GitLab installations, Redis logs are in `/var/log/gitlab/redis/current`. + +## Alertmanager Logs + +For Omnibus GitLab installations, Alertmanager logs are in `/var/log/gitlab/alertmanager/current`. + +<!-- vale gitlab.Spelling = NO --> + +## Crond Logs + +For Omnibus GitLab installations, crond logs are in `/var/log/gitlab/crond/`. + +<!-- vale gitlab.Spelling = YES --> + +## Grafana Logs + +For Omnibus GitLab installations, Grafana logs are in `/var/log/gitlab/grafana/current`. + +## LogRotate Logs + +For Omnibus GitLab installations, `logrotate` logs are in `/var/log/gitlab/logrotate/current`. + +## GitLab Monitor Logs + +For Omnibus GitLab installations, GitLab Monitor logs are in `/var/log/gitlab/gitlab-monitor/`. + +## GitLab Exporter + +For Omnibus GitLab installations, GitLab Exporter logs are in `/var/log/gitlab/gitlab-exporter/current`. + +## GitLab agent server + +For Omnibus GitLab installations, GitLab agent server logs are +in `/var/log/gitlab/gitlab-kas/current`. + +## Praefect Logs + +For Omnibus GitLab installations, Praefect logs are in `/var/log/gitlab/praefect/`. + +GitLab also tracks [Prometheus metrics for Praefect](../gitaly/monitoring.md#monitor-gitaly-cluster). + +## Backup log + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63832) in GitLab 14.1. + +For Omnibus installations, the backup log is located at `/var/log/gitlab/gitlab-rails/backup_json.log`. + +This log is populated when a [GitLab backup is created](../../raketasks/backup_restore.md). You can use this log to understand how the backup process performed. + +## Performance bar stats + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48149) in GitLab 13.7. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/performance_bar_json.log` +- Installations from source: `/home/git/gitlab/log/performance_bar_json.log` + +Performance bar statistics (currently only duration of SQL queries) are recorded +in that file. For example: + +```json +{"severity":"INFO","time":"2020-12-04T09:29:44.592Z","correlation_id":"33680b1490ccd35981b03639c406a697","filename":"app/models/ci/pipeline.rb","method_path":"app/models/ci/pipeline.rb:each_with_object","request_id":"rYHomD0VJS4","duration_ms":26.889,"count":2,"query_type": "active-record"} +``` + +These statistics are logged on .com only, disabled on self-deployments. + +## Gathering logs + +When [troubleshooting](../troubleshooting/index.md) issues that aren't localized to one of the +previously listed components, it's helpful to simultaneously gather multiple logs and statistics +from a GitLab instance. + +NOTE: +GitLab Support often asks for one of these, and maintains the required tools. + +### Briefly tail the main logs + +If the bug or error is readily reproducible, save the main GitLab logs +[to a file](../troubleshooting/linux_cheat_sheet.md#files-and-directories) while reproducing the +problem a few times: + +```shell +sudo gitlab-ctl tail | tee /tmp/<case-ID-and-keywords>.log +``` + +Conclude the log gathering with <kbd>Control</kbd> + <kbd>C</kbd>. + +### GitLabSOS + +If performance degradations or cascading errors occur that can't readily be attributed to one +of the previously listed GitLab components, [GitLabSOS](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) +can provide a broader perspective of the GitLab instance. For more details and instructions +to run it, read [the GitLabSOS documentation](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/#gitlabsos). + +### Fast-stats + +[Fast-stats](https://gitlab.com/gitlab-com/support/toolbox/fast-stats) is a tool +for creating and comparing performance statistics from GitLab logs. +For more details and instructions to run it, read the +[documentation for fast-stats](https://gitlab.com/gitlab-com/support/toolbox/fast-stats#usage). + +## Find relevant log entries with a correlation ID + +Most requests have a log ID that can be used to [find relevant log entries](tracing_correlation_id.md). diff --git a/doc/administration/logs/log_parsing.md b/doc/administration/logs/log_parsing.md new file mode 100644 index 00000000000..20b439af49b --- /dev/null +++ b/doc/administration/logs/log_parsing.md @@ -0,0 +1,316 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Parsing GitLab logs with `jq` **(FREE SELF)** + +We recommend using log aggregation and search tools like Kibana and Splunk whenever possible, +but if they are not available you can still quickly parse +[GitLab logs](../logs/index.md) in JSON format +(the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/). + +NOTE: +Specifically for summarizing error events and basic usage statistics, +the GitLab Support Team provides the specialised +[`fast-stats` tool](https://gitlab.com/gitlab-com/support/toolbox/fast-stats/#when-to-use-it). + +## What is JQ? + +As noted in its [manual](https://stedolan.github.io/jq/manual/), `jq` is a command-line JSON processor. The following examples +include use cases targeted for parsing GitLab log files. + +## Parsing Logs + +The examples listed below address their respective log files by +their relative Omnibus paths and default filenames. +Find the respective full paths in the [GitLab logs sections](../logs/index.md#production_jsonlog). + +### General Commands + +#### Pipe colorized `jq` output into `less` + +```shell +jq . <FILE> -C | less -R +``` + +#### Search for a term and pretty-print all matching lines + +```shell +grep <TERM> <FILE> | jq . +``` + +#### Skip invalid lines of JSON + +```shell +jq -cR 'fromjson?' file.json | jq <COMMAND> +``` + +By default `jq` errors out when it encounters a line that is not valid JSON. +This skips over all invalid lines and parses the rest. + +#### Print a JSON log's time range + +```shell +cat log.json | (head -1; tail -1) | jq '.time' +``` + +Use `zcat` if the file has been rotated and compressed: + +```shell +zcat @400000006026b71d1a7af804.s | (head -1; tail -1) | jq '.time' + +zcat some_json.log.25.gz | (head -1; tail -1) | jq '.time' +``` + +#### Get activity for correlation ID across multiple JSON logs in chronological order + +```shell +grep -hR <correlationID> | jq -c -R 'fromjson?' | jq -C -s 'sort_by(.time)' | less -R +``` + +### Parsing `gitlab-rails/production_json.log` and `gitlab-rails/api_json.log` + +#### Find all requests with a 5XX status code + +```shell +jq 'select(.status >= 500)' <FILE> +``` + +#### Top 10 slowest requests + +```shell +jq -s 'sort_by(-.duration_s) | limit(10; .[])' <FILE> +``` + +#### Find and pretty print all requests related to a project + +```shell +grep <PROJECT_NAME> <FILE> | jq . +``` + +#### Find all requests with a total duration > 5 seconds + +```shell +jq 'select(.duration_s > 5000)' <FILE> +``` + +#### Find all project requests with more than 5 rugged calls + +```shell +grep <PROJECT_NAME> <FILE> | jq 'select(.rugged_calls > 5)' +``` + +#### Find all requests with a Gitaly duration > 10 seconds + +```shell +jq 'select(.gitaly_duration_s > 10000)' <FILE> +``` + +#### Find all requests with a queue duration > 10 seconds + +```shell +jq 'select(.queue_duration_s > 10000)' <FILE> +``` + +#### Top 10 requests by # of Gitaly calls + +```shell +jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; .[])' <FILE> +``` + +### Parsing `gitlab-rails/production_json.log` + +#### Print the top three controller methods by request volume and their three longest durations + +```shell +jq -s -r 'group_by(.controller+.action) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tMETHOD: \(.[0].controller)#\(.[0].action)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' production_json.log +``` + +**Example output** + +```plaintext +CT: 2721 METHOD: SessionsController#new DURS: 844.06, 713.81, 704.66 +CT: 2435 METHOD: MetricsController#index DURS: 299.29, 284.01, 158.57 +CT: 1328 METHOD: Projects::NotesController#index DURS: 403.99, 386.29, 384.39 +``` + +### Parsing `gitlab-rails/api_json.log` + +#### Print top three routes with request count and their three longest durations + +```shell +jq -s -r 'group_by(.route) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tROUTE: \(.[0].route)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' api_json.log +``` + +**Example output** + +```plaintext +CT: 2472 ROUTE: /api/:version/internal/allowed DURS: 56402.65, 38411.43, 19500.41 +CT: 297 ROUTE: /api/:version/projects/:id/repository/tags DURS: 731.39, 685.57, 480.86 +CT: 190 ROUTE: /api/:version/projects/:id/repository/commits DURS: 1079.02, 979.68, 958.21 +``` + +### Print top API user agents + +```shell +jq --raw-output '[.route, .ua] | @tsv' api_json.log | sort | uniq -c | sort -n +``` + +**Example output**: + +```plaintext + 89 /api/:version/usage_data/increment_unique_users # plus browser details + 567 /api/:version/jobs/:id/trace gitlab-runner # plus version details +1234 /api/:version/internal/allowed GitLab-Shell +``` + +This sample response seems normal. A custom tool or script might be causing a high load +if the output contains many: + +- Third party libraries like `python-requests` or `curl`. +- [GitLab CLI clients](https://about.gitlab.com/partners/technology-partners/#cli-clients). + +You can also [use `fast-stats top`](#parsing-gitlab-logs-with-jq) to extract performance statistics. + +### Parsing `gitlab-workhorse/current` + +### Print top Workhorse user agents + +```shell +jq --raw-output '[.uri, .user_agent] | @tsv' current | sort | uniq -c | sort -n +``` + +**Example output**: + +```plaintext + 89 /api/graphql # plus browser details + 567 /api/v4/internal/allowed GitLab-Shell +1234 /api/v4/jobs/request gitlab-runner # plus version details +``` + +Similar to the [API `ua` data](#print-top-api-user-agents), +deviations from this common order might indicate scripts that could be optimized. + +The performance impact of runners checking for new jobs can be reduced by increasing +[the `check_interval` setting](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section), +for example. + +### Parsing `gitlab-rails/geo.log` + +#### Find most common Geo sync errors + +If [the `geo:status` Rake task](../geo/replication/troubleshooting.md#sync-status-rake-task) +repeatedly reports that some items never reach 100%, +the following command helps to focus on the most common errors. + +```shell +jq --raw-output 'select(.severity == "ERROR") | [.project_path, .message] | @tsv' geo.log | sort | uniq -c | sort | tail +``` + +### Parsing `gitaly/current` + +Use the following examples to [troubleshoot Gitaly](../gitaly/troubleshooting.md). + +#### Find all Gitaly requests sent from web UI + +```shell +jq 'select(."grpc.meta.client_name" == "gitlab-web")' current +``` + +#### Find all failed Gitaly requests + +```shell +jq 'select(."grpc.code" != null and ."grpc.code" != "OK")' current +``` + +#### Find all requests that took longer than 30 seconds + +```shell +jq 'select(."grpc.time_ms" > 30000)' current +``` + +#### Print top ten projects by request volume and their three longest durations + +```shell +jq --raw-output --slurp ' + map( + select( + ."grpc.request.glProjectPath" != null + and ."grpc.request.glProjectPath" != "" + and ."grpc.time_ms" != null + ) + ) + | group_by(."grpc.request.glProjectPath") + | sort_by(-length) + | limit(10; .[]) + | sort_by(-."grpc.time_ms") + | [ + length, + .[0]."grpc.time_ms", + .[1]."grpc.time_ms", + .[2]."grpc.time_ms", + .[0]."grpc.request.glProjectPath" + ] + | @sh' current \ +| awk 'BEGIN { printf "%7s %10s %10s %10s\t%s\n", "CT", "MAX DURS", "", "", "PROJECT" } + { printf "%7u %7u ms, %7u ms, %7u ms\t%s\n", $1, $2, $3, $4, $5 }' +``` + +**Example output** + +```plaintext + CT MAX DURS PROJECT + 206 4898 ms, 1101 ms, 1032 ms 'groupD/project4' + 109 1420 ms, 962 ms, 875 ms 'groupEF/project56' + 663 106 ms, 96 ms, 94 ms 'groupABC/project123' + ... +``` + +#### Find all projects affected by a fatal Git problem + +```shell +grep "fatal: " current | \ + jq '."grpc.request.glProjectPath"' | \ + sort | uniq +``` + +### Parsing `gitlab-shell/gitlab-shell.log` + +For investigating Git calls via SSH, from [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/367). + +Find the top 20 calls by project and user: + +```shell +jq --raw-output --slurp ' + map( + select( + .username != null and + .gl_project_path !=null + ) + ) + | group_by(.username+.gl_project_path) + | sort_by(-length) + | limit(20; .[]) + | "count: \(length)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ + gitlab-shell.log +``` + +Find the top 20 calls by project, user, and command: + +```shell +jq --raw-output --slurp ' + map( + select( + .command != null and + .username != null and + .gl_project_path !=null + ) + ) + | group_by(.username+.gl_project_path+.command) + | sort_by(-length) + | limit(20; .[]) + | "count: \(length)\tcommand: \(.[0].command)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ + gitlab-shell.log +``` diff --git a/doc/administration/logs/tracing_correlation_id.md b/doc/administration/logs/tracing_correlation_id.md new file mode 100644 index 00000000000..1d3e4955d3a --- /dev/null +++ b/doc/administration/logs/tracing_correlation_id.md @@ -0,0 +1,202 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Find relevant log entries with a correlation ID **(FREE SELF)** + +GitLab instances log a unique request tracking ID (known as the +"correlation ID") for most requests. Each individual request to GitLab gets +its own correlation ID, which then gets logged in each GitLab component's logs for that +request. This makes it easier to trace behavior in a +distributed system. Without this ID it can be difficult or +impossible to match correlating log entries. + +## Identify the correlation ID for a request + +The correlation ID is logged in structured logs under the key `correlation_id` +and in all response headers GitLab sends under the header `x-request-id`. +You can find your correlation ID by searching in either place. + +### Getting the correlation ID in your browser + +You can use your browser's developer tools to monitor and inspect network +activity with the site that you're visiting. See the links below for network monitoring +documentation for some popular browsers. + +- [Network Monitor - Firefox Developer Tools](https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor) +- [Inspect Network Activity In Chrome DevTools](https://developer.chrome.com/docs/devtools/network/) +- [Safari Web Development Tools](https://developer.apple.com/safari/tools/) +- [Microsoft Edge Network panel](https://docs.microsoft.com/en-us/microsoft-edge/devtools-guide-chromium/network/) + +To locate a relevant request and view its correlation ID: + +1. Enable persistent logging in your network monitor. Some actions in GitLab redirect you quickly after you submit a form, so this helps capture all relevant activity. +1. To help isolate the requests you are looking for, you can filter for `document` requests. +1. Select the request of interest to view further detail. +1. Go to the **Headers** section and look for **Response Headers**. There you should find an `x-request-id` header with a +value that was randomly generated by GitLab for the request. + +See the following example: + + + +### Getting the correlation ID from your logs + +Another approach to finding the correct correlation ID is to search or watch +your logs and find the `correlation_id` value for the log entry that you're +watching for. + +For example, let's say that you want learn what's happening or breaking when +you reproduce an action in GitLab. You could tail the GitLab logs, filtering +to requests by your user, and then watch the requests until you see what you're +interested in. + +### Getting the correlation ID from curl + +If you're using `curl` then you can use the verbose option to show request and response headers, as well as other debug information. + +```shell +➜ ~ curl --verbose "https://gitlab.example.com/api/v4/projects" +# look for a line that looks like this +< x-request-id: 4rAMkV3gof4 +``` + +#### Using jq + +This example uses [jq](https://stedolan.github.io/jq/) to filter results and +display values we most likely care about. + +```shell +sudo gitlab-ctl tail gitlab-rails/production_json.log | jq 'select(.username == "bob") | "User: \(.username), \(.method) \(.path), \(.controller)#\(.action), ID: \(.correlation_id)"' +``` + +```plaintext +"User: bob, GET /root/linux, ProjectsController#show, ID: U7k7fh6NpW3" +"User: bob, GET /root/linux/commits/master/signatures, Projects::CommitsController#signatures, ID: XPIHpctzEg1" +"User: bob, GET /root/linux/blob/master/README, Projects::BlobController#show, ID: LOt9hgi1TV4" +``` + +#### Using grep + +This example uses only `grep` and `tr`, which are more likely to be installed than `jq`. + +```shell +sudo gitlab-ctl tail gitlab-rails/production_json.log | grep '"username":"bob"' | tr ',' '\n' | egrep 'method|path|correlation_id' +``` + +```plaintext +{"method":"GET" +"path":"/root/linux" +"username":"bob" +"correlation_id":"U7k7fh6NpW3"} +{"method":"GET" +"path":"/root/linux/commits/master/signatures" +"username":"bob" +"correlation_id":"XPIHpctzEg1"} +{"method":"GET" +"path":"/root/linux/blob/master/README" +"username":"bob" +"correlation_id":"LOt9hgi1TV4"} +``` + +## Searching your logs for the correlation ID + +Once you have the correlation ID you can start searching for relevant log +entries. You can filter the lines by the correlation ID itself. +Combining a `find` and `grep` should be sufficient to find the entries you are looking for. + +```shell +# find <gitlab log directory> -type f -mtime -0 exec grep '<correlation ID>' '{}' '+' +find /var/log/gitlab -type f -mtime 0 -exec grep 'LOt9hgi1TV4' '{}' '+' +``` + +```plaintext +/var/log/gitlab/gitlab-workhorse/current:{"correlation_id":"LOt9hgi1TV4","duration_ms":2478,"host":"gitlab.domain.tld","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.tld/root/linux","remote_addr":"68.0.116.160:0","remote_ip":"[filtered]","status":200,"system":"http","time":"2019-09-17T22:17:19Z","uri":"/root/linux/blob/master/README?format=json\u0026viewer=rich","user_agent":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","written_bytes":1743} +/var/log/gitlab/gitaly/current:{"correlation_id":"LOt9hgi1TV4","grpc.code":"OK","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-web","grpc.method":"FindCommits","grpc.request.deadline":"2019-09-17T22:17:47Z","grpc.request.fullMethod":"/gitaly.CommitService/FindCommits","grpc.request.glProjectPath":"root/linux","grpc.request.glRepository":"project-1","grpc.request.repoPath":"@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git","grpc.request.repoStorage":"default","grpc.request.topLevelGroup":"@hashed","grpc.service":"gitaly.CommitService","grpc.start_time":"2019-09-17T22:17:17Z","grpc.time_ms":2319.161,"level":"info","msg":"finished streaming call with code OK","peer.address":"@","span.kind":"server","system":"grpc","time":"2019-09-17T22:17:19Z"} +/var/log/gitlab/gitlab-rails/production_json.log:{"method":"GET","path":"/root/linux/blob/master/README","format":"json","controller":"Projects::BlobController","action":"show","status":200,"duration":2448.77,"view":0.49,"db":21.63,"time":"2019-09-17T22:17:19.800Z","params":[{"key":"viewer","value":"rich"},{"key":"namespace_id","value":"root"},{"key":"project_id","value":"linux"},{"key":"id","value":"master/README"}],"remote_ip":"[filtered]","user_id":2,"username":"bob","ua":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","queue_duration":3.38,"gitaly_calls":1,"gitaly_duration":0.77,"rugged_calls":4,"rugged_duration_ms":28.74,"correlation_id":"LOt9hgi1TV4"} +``` + +### Searching in distributed architectures + +If you have done some horizontal scaling in your GitLab infrastructure, then +you must search across _all_ of your GitLab nodes. You can do this with +some sort of log aggregation software like Loki, ELK, Splunk, or others. + +You can use a tool like Ansible or PSSH (parallel SSH) that can execute identical commands across your servers in +parallel, or craft your own solution. + +### Viewing the request in the Performance Bar + +You can use the [performance bar](../monitoring/performance/performance_bar.md) to view interesting data including calls made to SQL and Gitaly. + +To view the data, the correlation ID of the request must match the same session as the user +viewing the performance bar. For API requests, this means that you must perform the request +using the session cookie of the signed-in user. + +For example, if you want to view the database queries executed for the following API endpoint: + +```shell +https://gitlab.com/api/v4/groups/2564205/projects?with_security_reports=true&page=1&per_page=1 +``` + +First, enable the **Developer Tools** panel. See [Getting the correlation ID in your browser](#getting-the-correlation-id-in-your-browser) for details on how to do this. + +After developer tools have been enabled, obtain a session cookie as follows: + +1. Visit <https://gitlab.com> while logged in. +1. Optional. Select **Fetch/XHR** request filter in the **Developer Tools** panel. This step is described for Google Chrome developer tools and is not strictly necessary, it just makes it easier to find the correct request. +1. Select the `results?request_id=<some-request-id>` request on the left hand side. +1. The session cookie is displayed under the `Request Headers` section of the `Headers` panel. Right-click on the cookie value and select `Copy value`. + + + +You have the value of the session cookie copied to your clipboard, for example: + +```shell +experimentation_subject_id=<subject-id>; _gitlab_session=<session-id>; event_filter=all; visitor_id=<visitor-id>; perf_bar_enabled=true; sidebar_collapsed=true; diff_view=inline; sast_entry_point_dismissed=true; auto_devops_settings_dismissed=true; cf_clearance=<cf-clearance>; collapsed_gutter=false; frequently_used_emojis=clap,thumbsup,rofl,tada,eyes,bow +``` + +Use the value of the session cookie to craft an API request by pasting it into a custom header of a `curl` request: + +```shell +$ curl --include "https://gitlab.com/api/v4/groups/2564205/projects?with_security_reports=true&page=1&per_page=1" \ +--header 'cookie: experimentation_subject_id=<subject-id>; _gitlab_session=<session-id>; event_filter=all; visitor_id=<visitor-id>; perf_bar_enabled=true; sidebar_collapsed=true; diff_view=inline; sast_entry_point_dismissed=true; auto_devops_settings_dismissed=true; cf_clearance=<cf-clearance>; collapsed_gutter=false; frequently_used_emojis=clap,thumbsup,rofl,tada,eyes,bow' + + date: Tue, 28 Sep 2021 03:55:33 GMT + content-type: application/json + ... + x-request-id: 01FGN8P881GF2E5J91JYA338Y3 + ... + [ + { + "id":27497069, + "description":"Analyzer for images used on live K8S containers based on Starboard" + }, + "container_registry_image_prefix":"registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning", + "..." + ] +``` + +The response contains the data from the API endpoint, and a `correlation_id` value, returned in the `x-request-id` header, as described in the [Identify the correlation ID for a request](#identify-the-correlation-id-for-a-request) section. + +You can then view the database details for this request: + +1. Paste the `x-request-id` value into the `request details` field of the [performance bar](../monitoring/performance/performance_bar.md) and press <kbd>Enter/Return</kbd>. This example uses the `x-request-id` value `01FGN8P881GF2E5J91JYA338Y3`, returned by the above response: + +  + +1. A new request is inserted into the `Request Selector` dropdown on the right-hand side of the Performance Bar. Select the new request to view the metrics of the API request: + +  + + <!-- vale gitlab.Substitutions = NO --> +1. Select the `pg` link in the Progress Bar to view the database queries executed by the API request: + +  + <!-- vale gitlab.Substitutions = YES --> + + The database query dialog is displayed: + +  diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index 9128f978dd5..00e28e650ea 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -135,6 +135,8 @@ For most JSON requests, `POST`, `PUT`, `PATCH`, and `DELETE` are blocked, and th even if they finish running on the GitLab Runner. - Jobs in the `running` state for longer than the project's time limit do not time out. - Pipelines cannot be started, retried or canceled. No new jobs can be created either. +- The status of the runners in `/admin/runners` won't be updated. +- `gitlab-runner verify` will return the error `ERROR: Verifying runner... is removed`. After Maintenance Mode is disabled, new jobs are picked up again. Jobs that were in the `running` state before enabling Maintenance Mode resume and their logs start diff --git a/doc/administration/monitoring/ip_allowlist.md b/doc/administration/monitoring/ip_allowlist.md index adf9516733a..3151b696182 100644 --- a/doc/administration/monitoring/ip_allowlist.md +++ b/doc/administration/monitoring/ip_allowlist.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Memory +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index d6575766b17..a2def8a9f64 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -111,6 +111,7 @@ The following metrics are available: | `failed_login_captcha_total` | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login | | | `successful_login_captcha_total` | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login | | | `auto_devops_pipelines_completed_total` | Counter | 12.7 | Counter of completed Auto DevOps pipelines, labeled by status | | +| `artifact_report_<report_type>_builds_completed_total` | Counter | 15.3 | Counter of completed CI Builds with report-type artifacts, grouped by report type and labeled by status | | | `gitlab_metrics_dashboard_processing_time_ms` | Summary | 12.10 | Metrics dashboard processing time in milliseconds | service, stages | | `action_cable_active_connections` | Gauge | 13.4 | Number of ActionCable WS clients currently connected | `server_mode` | | `action_cable_broadcasts_total` | Counter | 13.10 | The number of ActionCable broadcasts emitted | `server_mode` | @@ -184,6 +185,7 @@ configuration option in `gitlab.yml`. These metrics are served from the | `sidekiq_elasticsearch_requests_total` | Counter | 13.1 | Elasticsearch requests during a Sidekiq job execution | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` | | `sidekiq_running_jobs` | Gauge | 12.2 | Number of Sidekiq jobs running | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` | | `sidekiq_concurrency` | Gauge | 12.5 | Maximum number of Sidekiq jobs | | +| `sidekiq_mem_total_bytes` | Gauge | 15.3 | Number of bytes allocated for both objects consuming an object slot and objects that required a malloc'| | | `geo_db_replication_lag_seconds` | Gauge | 10.2 | Database replication lag (seconds) | `url` | | `geo_repositories` | Gauge | 10.2 | Total number of repositories available on primary | `url` | | `geo_repositories_synced` | Gauge | 10.2 | Number of repositories synced on secondary | `url` | @@ -300,6 +302,16 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads verifications failed on secondary | `url` | | `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` | | `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` | +| `geo_ci_secure_files` | Gauge | 15.3 | Number of secure files on primary | `url` | +| `geo_ci_secure_files_checksum_total` | Gauge | 15.3 | Number of secure files tried to checksum on primary | `url` | +| `geo_ci_secure_files_checksummed` | Gauge | 15.3 | Number of secure files successfully checksummed on primary | `url` | +| `geo_ci_secure_files_checksum_failed` | Gauge | 15.3 | Number of secure files failed to calculate the checksum on primary | `url` | +| `geo_ci_secure_files_synced` | Gauge | 15.3 | Number of syncable secure files synced on secondary | `url` | +| `geo_ci_secure_files_failed` | Gauge | 15.3 | Number of syncable secure files failed to sync on secondary | `url` | +| `geo_ci_secure_files_registry` | Gauge | 15.3 | Number of secure files in the registry | `url` | +| `geo_ci_secure_files_verification_total` | Gauge | 15.3 | Number of secure files verifications tried on secondary | `url` | +| `geo_ci_secure_files_verified` | Gauge | 15.3 | Number of secure files verified on secondary | `url` | +| `geo_ci_secure_files_verification_failed` | Gauge | 15.3 | Number of secure files verifications failed on secondary | `url` | ## Database load balancing metrics **(PREMIUM SELF)** @@ -378,8 +390,8 @@ Some basic Ruby runtime metrics are available: ## Redis metrics These client metrics are meant to complement Redis server metrics. -These metrics are broken down per [Redis -instance](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances). +These metrics are broken down per +[Redis instance](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances). These metrics all have a `storage` label which indicates the Redis instance (`cache`, `shared_state`, and so on). diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 51360800d66..6f6ac5c5d4b 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -465,3 +465,17 @@ If you are using Prometheus monitoring: ``` 1. Optional. [Configure the storage retention size](index.md#configure-the-storage-retention-size). + +### Monitoring node not receiving data + +If the monitoring node is not receiving any data, check that the exporters are capturing data: + +```shell +curl "http[s]://localhost:<EXPORTER LISTENING PORT>/metrics" +``` + +or + +```shell +curl "http[s]://localhost:<EXPORTER LISTENING PORT>/-/metrics" +``` diff --git a/doc/administration/monitoring/prometheus/web_exporter.md b/doc/administration/monitoring/prometheus/web_exporter.md index fe140b15ed2..45a8e8d5640 100644 --- a/doc/administration/monitoring/prometheus/web_exporter.md +++ b/doc/administration/monitoring/prometheus/web_exporter.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Memory +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- @@ -14,7 +14,7 @@ Prometheus scraper understands. NOTE: This page is about web application metrics. -To export background job metrics, learn how to [configure the Sidekiq metrics server](../../sidekiq.md#configure-the-sidekiq-metrics-server). +To export background job metrics, learn how to [configure the Sidekiq metrics server](../../sidekiq/index.md#configure-the-sidekiq-metrics-server). We provide two mechanisms by which web application metrics can be exported: diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index 51fa627b8d4..29f00fec3f7 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -65,8 +65,7 @@ Read: ## Known kernel version incompatibilities RedHat Enterprise Linux (RHEL) and CentOS v7.7 and v7.8 ship with kernel -version `3.10.0-1127`, which [contains a -bug](https://bugzilla.redhat.com/show_bug.cgi?id=1783554) that causes +version `3.10.0-1127`, which [contains a bug](https://bugzilla.redhat.com/show_bug.cgi?id=1783554) that causes [uploads to fail to copy over NFS](https://gitlab.com/gitlab-org/gitlab/-/issues/218999). The following GitLab versions include a fix to work properly with that kernel version: @@ -99,9 +98,20 @@ NFS performance with GitLab can in some cases be improved with [direct Git access](gitaly/index.md#direct-access-to-git-in-gitlab) using [Rugged](https://github.com/libgit2/rugged). -From GitLab 12.1, GitLab automatically detects if Rugged can and should be used per storage. -If you previously enabled Rugged using the feature flag and you want to use automatic detection instead, -you must unset the feature flag: +Versions of GitLab after 12.2 and prior to 15.3 automatically detect if +Rugged can and should be used per storage. + +NOTE: +GitLab 15.3 and later disables this automatic detection. Auto-detection can be enabled via the +`skip_rugged_auto_detect` feature flag: + +```ruby +Feature.disable(:skip_rugged_auto_detect) +``` + +In addition, if you previously enabled Rugged using the feature flag and +you want to use automatic detection instead, you must unset the feature +flag: ```shell sudo gitlab-rake gitlab:features:unset_rugged @@ -485,7 +495,7 @@ sudo perf trace --no-syscalls --event 'nfs4:*' -p $(pgrep -fd ',' puma) ### Warnings `garbage found: .../repositories/@hashed/...git/objects/pack/.nfs...` in Gitaly logs -If you find any warnings like `garbage found: .../repositories/@hashed/...git/objects/pack/.nfs...` in [Gitaly logs](logs.md#gitaly-logs), +If you find any warnings like `garbage found: .../repositories/@hashed/...git/objects/pack/.nfs...` in [Gitaly logs](logs/index.md#gitaly-logs), this problem occurs if `lookupcache=positive` is not set, which we recommend as an [NFS mount option](#mount-options). See [Gitaly issue #3175](https://gitlab.com/gitlab-org/gitaly/-/issues/3175) for more details. diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index ddeaf0280eb..0299d5f8b0c 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -26,8 +26,8 @@ GitLab has been tested by vendors and customers on a number of object storage pr ### Known compatibility issues -- Dell EMC ECS: Prior to GitLab 13.3, there is a [known bug in GitLab Workhorse that prevents - HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). +- Dell EMC ECS: Prior to GitLab 13.3, there is a + [known bug in GitLab Workhorse that prevents HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). Be sure to upgrade to GitLab 13.3.0 or above if you use S3 storage with this hardware. - Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying). @@ -578,9 +578,8 @@ real bucket into multiple virtual buckets. If your object storage bucket is called `my-gitlab-objects` you can configure uploads to go into `my-gitlab-objects/uploads`, artifacts into `my-gitlab-objects/artifacts`, etc. The application will act as if -these are separate buckets. Note that use of bucket prefixes [may not -work correctly with Helm -backups](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3376). +these are separate buckets. Note that use of bucket prefixes +[may not work correctly with Helm backups](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3376). Helm-based installs require separate buckets to [handle backup restorations](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-terraform-state-dependency-proxy). @@ -693,18 +692,17 @@ configuration. When configured either with an instance profile or with the consolidated object configuration, GitLab Workhorse properly uploads files to S3 -buckets that have [SSE-S3 or SSE-KMS encryption enabled by -default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). -Customer master keys (CMKs) and SSE-C encryption are [not -supported since this requires sending the encryption keys in every request](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). +buckets that have [SSE-S3 or SSE-KMS encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). +Customer master keys (CMKs) and SSE-C encryption are +[not supported since this requires sending the encryption keys in every request](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). ##### Server-side encryption headers > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38240) in GitLab 13.3. Setting a default encryption on an S3 bucket is the easiest way to -enable encryption, but you may want to [set a bucket policy to ensure -only encrypted objects are uploaded](https://aws.amazon.com/premiumsupport/knowledge-center/s3-bucket-store-kms-encrypted-objects/). +enable encryption, but you may want to +[set a bucket policy to ensure only encrypted objects are uploaded](https://aws.amazon.com/premiumsupport/knowledge-center/s3-bucket-store-kms-encrypted-objects/). To do this, you must configure GitLab to send the proper encryption headers in the `storage_options` configuration section: diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index 373017eefa7..58858c54843 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -1,362 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../sidekiq/extra_sidekiq_processes.md' +remove_date: '2022-11-11' --- -# Run multiple Sidekiq processes **(FREE SELF)** +This document was moved to [another location](../sidekiq/extra_sidekiq_processes.md). -GitLab allows you to start multiple Sidekiq processes. -These processes can be used to consume a dedicated set -of queues. This can be used to ensure certain queues always have dedicated -workers, no matter the number of jobs to be processed. - -NOTE: -The information in this page applies only to Omnibus GitLab. - -## Available Sidekiq queues - -For a list of the existing Sidekiq queues, check the following files: - -- [Queues for both GitLab Community and Enterprise Editions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) -- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml) - -Each entry in the above files represents a queue on which Sidekiq processes -can be started. - -## Start multiple processes - -> - [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4006) in GitLab 12.10, starting multiple processes with Sidekiq cluster. -> - [Sidekiq cluster moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. -> - [Sidekiq cluster became default](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4140) in GitLab 13.0. - -When starting multiple processes, the number of processes should -equal (and **not** exceed) the number of CPU cores you want to -dedicate to Sidekiq. Each Sidekiq process can use only 1 CPU -core, subject to the available workload and concurrency settings. - -To start multiple processes: - -1. Using the `sidekiq['queue_groups']` array setting, specify how many processes to - create using `sidekiq-cluster` and which queue they should handle. - Each item in the array equates to one additional Sidekiq - process, and values in each item determine the queues it works on. - - For example, the following setting creates three Sidekiq processes, one to run on - `elastic_commit_indexer`, one to run on `mailers`, and one process running on all queues: - - ```ruby - sidekiq['queue_groups'] = [ - "elastic_commit_indexer", - "mailers", - "*" - ] - ``` - - To have an additional Sidekiq process handle multiple queues, add multiple - queue names to its item delimited by commas. For example: - - ```ruby - sidekiq['queue_groups'] = [ - "elastic_commit_indexer, elastic_association_indexer", - "mailers", - "*" - ] - ``` - - [In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and - later, the special queue name `*` means all queues. This starts two - processes, each handling all queues: - - ```ruby - sidekiq['queue_groups'] = [ - "*", - "*" - ] - ``` - - `*` cannot be combined with concrete queue names - `*, mailers` - just handles the `mailers` queue. - - When `sidekiq-cluster` is only running on a single node, make sure that at least - one process is running on all queues using `*`. This ensures a process - automatically picks up jobs in queues created in the future, - including queues that have dedicated processes. - - If `sidekiq-cluster` is running on more than one node, you can also use - [`--negate`](#negate-settings) and list all the queues that are already being - processed. - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -To view the Sidekiq processes in GitLab: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Monitoring > Background Jobs**. - -## Negate settings - -To have the Sidekiq process work on every queue **except** the ones -you list. In this example, we exclude all import-related jobs from a Sidekiq node: - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['negate'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=importers" - ] - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Queue selector - -> - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in GitLab 12.8. -> - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. -> - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. - -In addition to selecting queues by name, as above, the `queue_selector` option -allows queue groups to be selected in a more general way using a [worker matching -query](extra_sidekiq_routing.md#worker-matching-query). After `queue_selector` -is set, all `queue_groups` must follow the aforementioned syntax. - -In `/etc/gitlab/gitlab.rb`: - -```ruby -sidekiq['enable'] = true -sidekiq['queue_selector'] = true -sidekiq['queue_groups'] = [ - # Run all non-CPU-bound queues that are high urgency - 'resource_boundary!=cpu&urgency=high', - # Run all continuous integration and pages queues that are not high urgency - 'feature_category=continuous_integration,pages&urgency!=high', - # Run all queues - '*' -] -``` - -## Ignore all import queues - -When [importing from GitHub](../../user/project/import/github.md) or -other sources, Sidekiq might use all of its resources to perform those -operations. To set up two separate `sidekiq-cluster` processes, where -one only processes imports and the other processes all other queues: - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=importers", - "feature_category!=importers" - ] - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Number of threads - -By default each process defined under `sidekiq` starts with a -number of threads that equals the number of queues, plus one spare thread. -For example, a process that handles the `process_commit` and `post_receive` -queues uses three threads in total. - -These thread run inside a single Ruby process, and each process -can only use a single CPU core. The usefulness of threading depends -on the work having some external dependencies to wait on, like database queries or -HTTP requests. Most Sidekiq deployments benefit from this threading, and when -running fewer queues in a process, increasing the thread count might be -even more desirable to make the most effective use of CPU resources. - -### Manage thread counts explicitly - -The correct maximum thread count (also called concurrency) depends on the workload. -Typical values range from `1` for highly CPU-bound tasks to `15` or higher for mixed -low-priority work. A reasonable starting range is `15` to `25` for a non-specialized -deployment. - -You can find example values used by GitLab.com by searching for `concurrency:` in -[the Helm charts](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/blob/master/releases/gitlab/values/gprd.yaml.gotmpl). -The values vary according to the work each specific deployment of Sidekiq does. -Any other specialized deployments with processes dedicated to specific queues should -have the concurrency tuned according to: -have the concurrency tuned according to: - -- The CPU usage of each type of process. -- The throughput achieved. - -Each thread requires a Redis connection, so adding threads may increase Redis -latency and potentially cause client timeouts. See the [Sidekiq documentation -about Redis](https://github.com/mperham/sidekiq/wiki/Using-Redis) for more -details. - -#### When running Sidekiq cluster (default) - -Running Sidekiq cluster is the default in GitLab 13.0 and later. - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['min_concurrency'] = 15 - sidekiq['max_concurrency'] = 25 - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -`min_concurrency` and `max_concurrency` are independent; one can be set without -the other. Setting `min_concurrency` to `0` disables the limit. - -For each queue group, let `N` be one more than the number of queues. The -concurrency is set to: - -1. `N`, if it's between `min_concurrency` and `max_concurrency`. -1. `max_concurrency`, if `N` exceeds this value. -1. `min_concurrency`, if `N` is less than this value. - -If `min_concurrency` is equal to `max_concurrency`, then this value is used -regardless of the number of queues. - -When `min_concurrency` is greater than `max_concurrency`, it is treated as -being equal to `max_concurrency`. - -#### When running a single Sidekiq process - -Running a single Sidekiq process is the default in GitLab 12.10 and earlier. - -WARNING: -Running Sidekiq directly was removed in GitLab -[14.0](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240). - -1. Edit `/etc/gitlab/gitlab.rb` and add: - - ```ruby - sidekiq['cluster'] = false - sidekiq['concurrency'] = 25 - ``` - -1. Save the file and reconfigure GitLab for the changes to take effect: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -This sets the concurrency (number of threads) for the Sidekiq process. - -## Modify the check interval - -To modify `sidekiq-cluster`'s health check interval for the additional Sidekiq processes: - -1. Edit `/etc/gitlab/gitlab.rb` and add (the value can be any integer number of seconds): - - ```ruby - sidekiq['interval'] = 5 - ``` - -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - -## Troubleshoot using the CLI - -WARNING: -It's recommended to use `/etc/gitlab/gitlab.rb` to configure the Sidekiq processes. -If you experience a problem, you should contact GitLab support. Use the command -line at your own risk. - -For debugging purposes, you can start extra Sidekiq processes by using the command -`/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster`. This command -takes arguments using the following syntax: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster [QUEUE,QUEUE,...] [QUEUE, ...] -``` - -Each separate argument denotes a group of queues that have to be processed by a -Sidekiq process. Multiple queues can be processed by the same process by -separating them with a comma instead of a space. - -Instead of a queue, a queue namespace can also be provided, to have the process -automatically listen on all queues in that namespace without needing to -explicitly list all the queue names. For more information about queue namespaces, -see the relevant section in the -[Sidekiq development documentation](../../development/sidekiq/index.md#queue-namespaces). - -For example, say you want to start 2 extra processes: one to process the -`process_commit` queue, and one to process the `post_receive` queue. This can be -done as follows: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit post_receive -``` - -If you instead want to start one process processing both queues, you'd use the -following syntax: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive -``` - -If you want to have one Sidekiq process dealing with the `process_commit` and -`post_receive` queues, and one process to process the `gitlab_shell` queue, -you'd use the following: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive gitlab_shell -``` - -### Monitor the `sidekiq-cluster` command - -The `sidekiq-cluster` command does not terminate once it has started the desired -amount of Sidekiq processes. Instead, the process continues running and -forwards any signals to the child processes. This allows you to stop all -Sidekiq processes as you send a signal to the `sidekiq-cluster` process, -instead of having to send it to the individual processes. - -If the `sidekiq-cluster` process crashes or receives a `SIGKILL`, the child -processes terminate themselves after a few seconds. This ensures you don't -end up with zombie Sidekiq processes. - -This allows you to monitor the processes by hooking up -`sidekiq-cluster` to your supervisor of choice (for example, runit). - -If a child process died the `sidekiq-cluster` command signals all remaining -process to terminate, then terminate itself. This removes the need for -`sidekiq-cluster` to re-implement complex process monitoring/restarting code. -Instead you should make sure your supervisor restarts the `sidekiq-cluster` -process whenever necessary. - -### PID files - -The `sidekiq-cluster` command can store its PID in a file. By default no PID -file is written, but this can be changed by passing the `--pidfile` option to -`sidekiq-cluster`. For example: - -```shell -/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit -``` - -Keep in mind that the PID file contains the PID of the `sidekiq-cluster` -command and not the PIDs of the started Sidekiq processes. - -### Environment - -The Rails environment can be set by passing the `--environment` flag to the -`sidekiq-cluster` command, or by setting `RAILS_ENV` to a non-empty value. The -default value can be found in `/opt/gitlab/etc/gitlab-rails/env/RAILS_ENV`. +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/operations/extra_sidekiq_routing.md b/doc/administration/operations/extra_sidekiq_routing.md index a6ad3e62bb7..072b6f63537 100644 --- a/doc/administration/operations/extra_sidekiq_routing.md +++ b/doc/administration/operations/extra_sidekiq_routing.md @@ -1,193 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../sidekiq/extra_sidekiq_routing.md' +remove_date: '2022-11-11' --- -# Queue routing rules **(FREE SELF)** +This document was moved to [another location](../sidekiq/extra_sidekiq_routing.md). -When the number of Sidekiq jobs increases to a certain scale, the system faces -some scalability issues. One of them is that the length of the queue tends to get -longer. High-urgency jobs have to wait longer until other less urgent jobs -finish. This head-of-line blocking situation may eventually affect the -responsiveness of the system, especially critical actions. In another scenario, -the performance of some jobs is degraded due to other long running or CPU-intensive jobs -(computing or rendering ones) in the same machine. - -To counter the aforementioned issues, one effective solution is to split -Sidekiq jobs into different queues and assign machines handling each queue -exclusively. For example, all CPU-intensive jobs could be routed to the -`cpu-bound` queue and handled by a fleet of CPU optimized instances. The queue -topology differs between companies depending on the workloads and usage -patterns. Therefore, GitLab supports a flexible mechanism for the -administrator to route the jobs based on their characteristics. - -As an alternative to [Queue selector](extra_sidekiq_processes.md#queue-selector), which -configures Sidekiq cluster to listen to a specific set of workers or queues, -GitLab also supports routing a job from a worker to the desired queue when it -is scheduled. Sidekiq clients try to match a job against a configured list of -routing rules. Rules are evaluated from first to last, and as soon as we find a -match for a given worker we stop processing for that worker (first match wins). -If the worker doesn't match any rule, it falls back to the queue name generated -from the worker name. - -By default, if the routing rules are not configured (or denoted with an empty -array), all the jobs are routed to the queue generated from the worker name. - -## Example configuration - -In `/etc/gitlab/gitlab.rb`: - -```ruby -sidekiq['routing_rules'] = [ - # Do not re-route workers that require their own queue - ['tags=needs_own_queue', nil], - # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue - ['resource_boundary!=cpu&urgency=high', 'high-urgency'], - # Route all database, gitaly and global search workers that are throttled to `throttled` queue - ['feature_category=database,gitaly,global_search&urgency=throttled', 'throttled'], - # Route all workers having contact with outside work to a `network-intenstive` queue - ['has_external_dependencies=true|feature_category=hooks|tags=network', 'network-intensive'], - # Route all import workers to the queues generated by the worker name, for - # example, JiraImportWorker to `jira_import`, SVNWorker to `svn_worker` - ['feature_category=import', nil], - # Wildcard matching, route the rest to `default` queue - ['*', 'default'] -] -``` - -The routing rules list is an order-matter array of tuples of query and -corresponding queue: - -- The query is following a [worker matching query](#worker-matching-query) syntax. -- The `<queue_name>` must be a valid Sidekiq queue name. If the queue name - is `nil`, or an empty string, the worker is routed to the queue generated - by the name of the worker instead. - -The query supports wildcard matching `*`, which matches all workers. As a -result, the wildcard query must stay at the end of the list or the rules after it -are ignored. - -NOTE: -Mixing queue routing rules and queue selectors requires care to -ensure all jobs that are scheduled and picked up by appropriate Sidekiq -workers. - -## Worker matching query - -GitLab provides a query syntax to match a worker based on its -attributes. This query syntax is employed by both [Queue routing -rules](#queue-routing-rules) and [Queue -selector](extra_sidekiq_processes.md#queue-selector). A query includes two -components: - -- Attributes that can be selected. -- Operators used to construct a query. - -### Available attributes - -> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). - -Queue matching query works upon the worker attributes, described in -[Sidekiq style guide](../../development/sidekiq/index.md). We support querying -based on a subset of worker attributes: - -- `feature_category` - the [GitLab feature - category](https://about.gitlab.com/direction/maturity/#category-maturity) the - queue belongs to. For example, the `merge` queue belongs to the - `source_code_management` category. -- `has_external_dependencies` - whether or not the queue connects to external - services. For example, all importers have this set to `true`. -- `urgency` - how important it is that this queue's jobs run - quickly. Can be `high`, `low`, or `throttled`. For example, the - `authorized_projects` queue is used to refresh user permissions, and - is `high` urgency. -- `worker_name` - the worker name. Use this attribute to select a specific worker. -- `name` - the queue name generated from the worker name. Use this attribute to select a specific queue. Because this is generated from - the worker name, it does not change based on the result of other routing - rules. -- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or - `unknown`. For example, the `ProjectExportWorker` is memory bound as it has - to load data in memory before saving it for export. -- `tags` - short-lived annotations for queues. These are expected to frequently - change from release to release, and may be removed entirely. - -`has_external_dependencies` is a boolean attribute: only the exact -string `true` is considered true, and everything else is considered -false. - -`tags` is a set, which means that `=` checks for intersecting sets, and -`!=` checks for disjoint sets. For example, `tags=a,b` selects queues -that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have -neither of those tags. - -The attributes of each worker are hard-coded in the source code. For -convenience, we generate a [list of all available attributes in -GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) -and a [list of all available attributes in -GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml). - -### Available operators - -`queue_selector` supports the following operators, listed from highest -to lowest precedence: - -- `|` - the logical `OR` operator. For example, `query_a|query_b` (where `query_a` - and `query_b` are queries made up of the other operators here) includes - queues that match either query. -- `&` - the logical `AND` operator. For example, `query_a&query_b` (where - `query_a` and `query_b` are queries made up of the other operators here) will - only include queues that match both queries. -- `!=` - the `NOT IN` operator. For example, `feature_category!=issue_tracking` - excludes all queues from the `issue_tracking` feature category. -- `=` - the `IN` operator. For example, `resource_boundary=cpu` includes all - queues that are CPU bound. -- `,` - the concatenate set operator. For example, - `feature_category=continuous_integration,pages` includes all queues from - either the `continuous_integration` category or the `pages` category. This - example is also possible using the OR operator, but allows greater brevity, as - well as being lower precedence. - -The operator precedence for this syntax is fixed: it's not possible to make `AND` -have higher precedence than `OR`. - -[In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and -later, as with the standard queue group syntax above, a single `*` as the -entire queue group selects all queues. - -### Migration - -After the Sidekiq routing rules are changed, administrators must take care -with the migration to avoid losing jobs entirely, especially in a system with -long queues of jobs. The migration can be done by following the migration steps -mentioned in [Sidekiq job -migration](../../raketasks/sidekiq_job_migration.md) - -### Workers that cannot be migrated - -Some workers cannot share a queue with other workers - typically because -they check the size of their own queue - and so must be excluded from -this process. We recommend excluding these from any further worker -routing by adding a rule to keep them in their own queue, for example: - -```ruby -sidekiq['routing_rules'] = [ - ['tags=needs_own_queue', nil], - # ... -] -``` - -These queues must also be included in at least one [Sidekiq -queue group](extra_sidekiq_processes.md#start-multiple-processes). - -The following table shows the workers that should have their own queue: - -| Worker name | Queue name | GitLab issue | -| --- | --- | --- | -| `EmailReceiverWorker` | `email_receiver` | [`gitlab-com/gl-infra/scalability#1263`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) | -| `ServiceDeskEmailReceiverWorker` | `service_desk_email_receiver` | [`gitlab-com/gl-infra/scalability#1263`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1263) | -| `ProjectImportScheduleWorker` | `project_import_schedule` | [`gitlab-org/gitlab#340630`](https://gitlab.com/gitlab-org/gitlab/-/issues/340630) | -| `HashedStorage::MigratorWorker` | `hashed_storage:hashed_storage_migrator` | [`gitlab-org/gitlab#340629`](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | -| `HashedStorage::ProjectMigrateWorker` | `hashed_storage:hashed_storage_project_migrate` | [`gitlab-org/gitlab#340629`](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | -| `HashedStorage::ProjectRollbackWorker` | `hashed_storage:hashed_storage_project_rollback` | [`gitlab-org/gitlab#340629`](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | -| `HashedStorage::RollbackerWorker` | `hashed_storage:hashed_storage_rollbacker` | [`gitlab-org/gitlab#340629`](https://gitlab.com/gitlab-org/gitlab/-/issues/340629) | +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index 41a9a0e6d67..a6e66abdbdb 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -18,10 +18,9 @@ Keep your GitLab instance up and running smoothly. - [Multiple Sidekiq processes](extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that must be processed. **(FREE SELF)** - [Sidekiq routing rules](extra_sidekiq_routing.md): Configure the routing rules to route a job from a worker to a desirable queue. **(FREE SELF)** - [Puma](puma.md): Understand Puma and puma-worker-killer. -- Speed up SSH operations by [Authorizing SSH users via a fast, - indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or - by [doing away with user SSH keys stored on GitLab entirely in favor - of SSH certificates](ssh_certificates.md). +- Speed up SSH operations by + [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or + by [doing away with user SSH keys stored on GitLab entirely in favor of SSH certificates](ssh_certificates.md). - [File System Performance Benchmarking](filesystem_benchmarking.md): File system performance can have a big impact on GitLab performance, especially for actions that read or write Git repositories. This information helps benchmark diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index bc95ee626ce..5ce469d3e63 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -54,7 +54,7 @@ A higher value of `1200` or more would be beneficial if the server has free memo The worker killer checks memory every 20 seconds. -To monitor the worker killer, use [the Puma log](../logs.md#puma_stdoutlog) `/var/log/gitlab/puma/puma_stdout.log`. +To monitor the worker killer, use [the Puma log](../logs/index.md#puma_stdoutlog) `/var/log/gitlab/puma/puma_stdout.log`. For example: ```plaintext @@ -246,7 +246,7 @@ automatically, due to differences between the two application servers. To switch from Unicorn to Puma: 1. Determine suitable Puma [worker and thread settings](../../install/requirements.md#puma-settings). -1. Convert any custom Unicorn settings to Puma. +1. Convert any custom Unicorn settings to Puma in `/etc/gitlab/gitlab.rb`. The table below summarizes which Unicorn configuration keys correspond to those in Puma when using the Linux package, and which ones have no corresponding counterpart. diff --git a/doc/administration/operations/rails_console.md b/doc/administration/operations/rails_console.md index 660a99faaf8..430dfbc637c 100644 --- a/doc/administration/operations/rails_console.md +++ b/doc/administration/operations/rails_console.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Rails console **(FREE SELF)** -At the heart of GitLab is a web application [built using the Ruby on Rails -framework](https://about.gitlab.com/blog/2018/10/29/why-we-use-rails-to-build-gitlab/). +At the heart of GitLab is a web application +[built using the Ruby on Rails framework](https://about.gitlab.com/blog/2018/10/29/why-we-use-rails-to-build-gitlab/). The [Rails console](https://guides.rubyonrails.org/command_line.html#rails-console). provides a way to interact with your GitLab instance from the command line, and also grants access to the amazing tools built right into Rails. @@ -19,8 +19,8 @@ with no consequences, you are strongly advised to do so in a test environment. The Rails console is for GitLab system administrators who are troubleshooting a problem or need to retrieve some data that can only be done through direct -access of the GitLab application. Basic knowledge of Ruby is needed (try [this -30-minute tutorial](https://try.ruby-lang.org/) for a quick introduction). +access of the GitLab application. Basic knowledge of Ruby is needed (try +[this 30-minute tutorial](https://try.ruby-lang.org/) for a quick introduction). Rails experience is useful but not required. ## Starting a Rails console session @@ -136,8 +136,8 @@ root 1 ``` -Some basic knowledge of Ruby will be very useful. Try [this -30-minute tutorial](https://try.ruby-lang.org/) for a quick introduction. +Some basic knowledge of Ruby will be very useful. Try +[this 30-minute tutorial](https://try.ruby-lang.org/) for a quick introduction. Rails experience is helpful but not essential. ### Troubleshooting Rails Runner diff --git a/doc/administration/operations/sidekiq_memory_killer.md b/doc/administration/operations/sidekiq_memory_killer.md index d9558c3d7a6..b1977589fae 100644 --- a/doc/administration/operations/sidekiq_memory_killer.md +++ b/doc/administration/operations/sidekiq_memory_killer.md @@ -1,83 +1,11 @@ --- -stage: Data Stores -group: Memory -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../sidekiq/sidekiq_memory_killer.md' +remove_date: '2022-11-11' --- -# Sidekiq MemoryKiller **(FREE SELF)** +This document was moved to [another location](../sidekiq/sidekiq_memory_killer.md). -The GitLab Rails application code suffers from memory leaks. For web requests -this problem is made manageable using -[`puma-worker-killer`](https://github.com/schneems/puma_worker_killer) which -restarts Puma worker processes if it exceeds a memory limit. The Sidekiq -MemoryKiller applies the same approach to the Sidekiq processes used by GitLab -to process background jobs. - -Unlike puma-worker-killer, which is enabled by default for all GitLab -installations of GitLab 13.0 and later, the Sidekiq MemoryKiller is enabled by default -_only_ for Omnibus packages. The reason for this is that the MemoryKiller -relies on runit to restart Sidekiq after a memory-induced shutdown and GitLab -installations from source do not all use runit or an equivalent. - -With the default settings, the MemoryKiller causes a Sidekiq restart no -more often than once every 15 minutes, with the restart causing about one -minute of delay for incoming background jobs. - -Some background jobs rely on long-running external processes. To ensure these -are cleanly terminated when Sidekiq is restarted, each Sidekiq process should be -run as a process group leader (for example, using `chpst -P`). If using Omnibus or the -`bin/background_jobs` script with `runit` installed, this is handled for you. - -## Configuring the MemoryKiller - -The MemoryKiller is controlled using environment variables. - -- `SIDEKIQ_DAEMON_MEMORY_KILLER`: defaults to 1. When set to 0, the MemoryKiller - works in _legacy_ mode. Otherwise, the MemoryKiller works in _daemon_ mode. - - In _legacy_ mode, the MemoryKiller checks the Sidekiq process RSS - ([Resident Set Size](https://github.com/mperham/sidekiq/wiki/Memory#rss)) - after each job. - - In _daemon_ mode, the MemoryKiller checks the Sidekiq process RSS every 3 seconds - (defined by `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL`). - -- `SIDEKIQ_MEMORY_KILLER_MAX_RSS` (KB): if this variable is set, and its value is greater - than 0, the MemoryKiller is enabled. Otherwise the MemoryKiller is disabled. - - `SIDEKIQ_MEMORY_KILLER_MAX_RSS` defines the Sidekiq process allowed RSS. - - In _legacy_ mode, if the Sidekiq process exceeds the allowed RSS then an irreversible - delayed graceful restart is triggered. The restart of Sidekiq happens - after `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` seconds. - - In _daemon_ mode, if the Sidekiq process exceeds the allowed RSS for longer than - `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` the graceful restart is triggered. If the - Sidekiq process go below the allowed RSS within `SIDEKIQ_MEMORY_KILLER_GRACE_TIME`, - the restart is aborted. - - The default value for Omnibus packages is set - [in the Omnibus GitLab - repository](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb). - -- `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS` (KB): is used by _daemon_ mode. If the Sidekiq - process RSS (expressed in kilobytes) exceeds `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS`, - an immediate graceful restart of Sidekiq is triggered. - -- `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL`: used in _daemon_ mode to define how - often to check process RSS, default to 3 seconds. - -- `SIDEKIQ_MEMORY_KILLER_GRACE_TIME`: defaults to 900 seconds (15 minutes). - The usage of this variable is described as part of `SIDEKIQ_MEMORY_KILLER_MAX_RSS`. - -- `SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT`: defaults to 30 seconds. This defines the - maximum time allowed for all Sidekiq jobs to finish. No new jobs are accepted - during that time, and the process exits as soon as all jobs finish. - - If jobs do not finish during that time, the MemoryKiller interrupts all currently - running jobs by sending `SIGTERM` to the Sidekiq process. - - If the process hard shutdown/restart is not performed by Sidekiq, - the Sidekiq process is forcefully terminated after - `Sidekiq.options[:timeout] + 2` seconds. An external supervision mechanism - (for example, runit) must restart Sidekiq afterwards. +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/operations/ssh_certificates.md b/doc/administration/operations/ssh_certificates.md index a777f5484fd..1e405189342 100644 --- a/doc/administration/operations/ssh_certificates.md +++ b/doc/administration/operations/ssh_certificates.md @@ -35,10 +35,10 @@ uploading user SSH keys to GitLab entirely. ## Setting up SSH certificate lookup via GitLab Shell How to fully set up SSH certificates is outside the scope of this -document. See [OpenSSH's -`PROTOCOL.certkeys`](https://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.certkeys?annotate=HEAD) -for how it works, for example [RedHat's documentation about -it](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-using_openssh_certificate_authentication). +document. See +[OpenSSH's`PROTOCOL.certkeys`](https://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.certkeys?annotate=HEAD) +for how it works, for example +[RedHat's documentation about it](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/sec-using_openssh_certificate_authentication). We assume that you already have SSH certificates set up, and have added the `TrustedUserCAKeys` of your CA to your `sshd_config`, for example: @@ -143,8 +143,7 @@ This is because if the `AuthorizedPrincipalsCommand` can't authenticate the user, OpenSSH falls back on `~/.ssh/authorized_keys` (or the `AuthorizedKeysCommand`). -Therefore there may still be a reason to use the ["Fast lookup of -authorized SSH keys in the database"](fast_ssh_key_lookup.html) method +Therefore there may still be a reason to use the [Fast lookup of authorized SSH keys in the database](fast_ssh_key_lookup.md) method in conjunction with this. Since you are using SSH certificates for all your normal users, and relying on the `~/.ssh/authorized_keys` fallback for deploy keys, if you make use of those. @@ -160,8 +159,8 @@ users (especially if they're renewed) than you have deploy keys. Users can still bypass SSH certificate authentication by manually uploading an SSH public key to their profile, relying on the `~/.ssh/authorized_keys` fallback to authenticate it. There's -currently no feature to prevent this, [but there's an open request for -adding it](https://gitlab.com/gitlab-org/gitlab/-/issues/23260). +currently no feature to prevent this, +[but there's an open request for adding it](https://gitlab.com/gitlab-org/gitlab/-/issues/23260). Such a restriction can currently be hacked in by, for example, providing a custom `AuthorizedKeysCommand` which checks if the discovered key-ID diff --git a/doc/administration/package_information/deprecation_policy.md b/doc/administration/package_information/deprecation_policy.md index 422339c014c..73059514e1c 100644 --- a/doc/administration/package_information/deprecation_policy.md +++ b/doc/administration/package_information/deprecation_policy.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Deprecation policy **(FREE SELF)** +# Omnibus GitLab deprecation policy **(FREE SELF)** The Omnibus GitLab packages come with number of different libraries and services which offers users plethora of configuration options. diff --git a/doc/administration/package_information/index.md b/doc/administration/package_information/index.md index 746961e8613..d3a82b87116 100644 --- a/doc/administration/package_information/index.md +++ b/doc/administration/package_information/index.md @@ -34,8 +34,8 @@ These defaults are noted in the package [defaults document](defaults.md). ## Checking the versions of bundled software -After the Omnibus GitLab package is installed, all versions of the bundled -libraries are located in `/opt/gitlab/version-manifest.txt`. +After the Omnibus GitLab package is installed, you can find the version of +GitLab and all bundled libraries in `/opt/gitlab/version-manifest.txt`. If you don't have the package installed, you can always check the Omnibus GitLab [source repository](https://gitlab.com/gitlab-org/omnibus-gitlab/tree/master), specifically the diff --git a/doc/administration/package_information/postgresql_versions.md b/doc/administration/package_information/postgresql_versions.md index afdc63c4851..d831e8cea7f 100644 --- a/doc/administration/package_information/postgresql_versions.md +++ b/doc/administration/package_information/postgresql_versions.md @@ -11,15 +11,16 @@ This table lists only GitLab versions where a significant change happened in the package regarding PostgreSQL versions, not all. Usually, PostgreSQL versions change with major or minor GitLab releases. However, patch versions -of Omnibus GitLab sometimes update the patch level of PostgreSQL. +of Omnibus GitLab sometimes update the patch level of PostgreSQL. We've established a +[yearly cadence for PostgreSQL upgrades](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/postgresql-upgrade-cadence.html) +and trigger automatic database upgrades in the release before the new version is required. For example: - Omnibus 12.7.6 shipped with PostgreSQL 9.6.14 and 10.9. - Omnibus 12.7.7 shipped with PostgreSQL 9.6.17 and 10.12. -[Find out which versions of PostgreSQL (and other components) ship with -each Omnibus GitLab release](https://gitlab-org.gitlab.io/omnibus-gitlab/licenses.html). +[Find out which versions of PostgreSQL (and other components) ship with each Omnibus GitLab release](https://gitlab-org.gitlab.io/omnibus-gitlab/licenses.html). The lowest supported PostgreSQL versions are listed in the [installation requirements](../../install/requirements.md#postgresql-requirements). diff --git a/doc/administration/package_information/supported_os.md b/doc/administration/package_information/supported_os.md index 4f31981a05c..e4d7ea72cdc 100644 --- a/doc/administration/package_information/supported_os.md +++ b/doc/administration/package_information/supported_os.md @@ -10,8 +10,8 @@ GitLab officially supports LTS versions of operating systems. While OSs like Ubuntu have a clear distinction between LTS and non-LTS versions, there are other OSs, openSUSE for example, that don't follow the LTS concept. Hence to avoid confusion, the official policy is that at any point of time, all the -operating systems supported by GitLab are listed in the [installation -page](https://about.gitlab.com/install/). +operating systems supported by GitLab are listed in the +[installation page](https://about.gitlab.com/install/). The following lists the currently supported OSs and their possible EOL dates. @@ -19,11 +19,10 @@ The following lists the currently supported OSs and their possible EOL dates. | ------------------------------------------------------------ | ------------------------------ | --------------- | :----------------------------------------------------------: | ---------- | ------------------------------------------------------------ | | AlmaLinux 8 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [AlmaLinux Install Documentation](https://about.gitlab.com/install/#almalinux-8) | 2029 | <https://almalinux.org/> | | CentOS 7 | GitLab CE / GitLab EE 7.10.0 | x86_64 | [CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | June 2024 | <https://wiki.centos.org/About/Product> | -| Debian 9 | GitLab CE / GitLab EE 9.3.0 | amd64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2022 | <https://wiki.debian.org/LTS> | | Debian 10 | GitLab CE / GitLab EE 12.2.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2024 | <https://wiki.debian.org/LTS> | | Debian 11 | GitLab CE / GitLab EE 14.6.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2026 | <https://wiki.debian.org/LTS> | | OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap-15-3) | Nov 2022 | <https://en.opensuse.org/Lifetime> | -| RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | May 2024 | [RHEL Details](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates) | +| RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | May 2029 | [RHEL Details](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates) | | SLES 12 | GitLab EE 9.0.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap-15-3) | Oct 2027 | <https://www.suse.com/lifecycle/> | | Oracle Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | Jul 2024 | <https://www.oracle.com/a/ocom/docs/elsp-lifetime-069338.pdf> | | Scientific Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | June 2024 | <https://scientificlinux.org/downloads/sl-versions/sl7/> | @@ -87,6 +86,7 @@ release for them can be found below: | OpenSUSE 15.1 | [November 2020](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-13.12&dist=opensuse%2F15.1) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-13.12&dist=opensuse%2F15.1) 13.12 | | Ubuntu 16.04 | [April 2021](https://ubuntu.com/info/release-end-of-life) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_13.12&dist=ubuntu%2Fxenial) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_13.12&dist=ubuntu%2Fxenial) 13.12 | | OpenSUSE 15.2 | [December 2021](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-14.7&dist=opensuse%2F15.2) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-14.7&dist=opensuse%2F15.2) 14.7 | +| Debian 9 "Stretch" | [June 2022](https://lists.debian.org/debian-lts-announce/2022/07/msg00002.html) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_15.2&dist=debian%2Fstretch) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_15.2&dist=debian%2Fstretch) 15.2 | NOTE: An exception to this deprecation policy is when we are unable to provide diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 3bfb3ceca86..8b115ca1af4 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -739,7 +739,7 @@ auth: Without these entries, the registry logins cannot authenticate with GitLab. GitLab also remains unaware of -[nested image names](../../user/packages/container_registry/#image-naming-convention) +[nested image names](../../user/packages/container_registry/index.md#image-naming-convention) under the project hierarchy, like `registry.example.com/group/project/image-name:tag` or `registry.example.com/group/project/my/image-name:tag`, and only recognizes @@ -1202,8 +1202,8 @@ Before diving in to the following sections, here's some basic troubleshooting: been synchronized (for example, via NTP). 1. If you are using an S3-backed Registry, double check that the IAM - permissions and the S3 credentials (including region) are correct. See [the - sample IAM policy](https://docs.docker.com/registry/storage-drivers/s3/) + permissions and the S3 credentials (including region) are correct. See + [the sample IAM policy](https://docs.docker.com/registry/storage-drivers/s3/) for more details. 1. Check the Registry logs (for example `/var/log/gitlab/registry/current`) and the GitLab production logs @@ -1388,7 +1388,7 @@ project or branch name. Special characters can include: - Trailing hyphen/dash - Double hyphen/dash -To get around this, you can [change the group path](../../user/group/index.md#change-a-groups-path), +To get around this, you can [change the group path](../../user/group/manage.md#change-a-groups-path), [change the project path](../../user/project/settings/index.md#rename-a-repository) or change the branch name. Another option is to create a [push rule](../../user/project/repository/push_rules.md) to prevent this at the instance level. @@ -1631,8 +1631,8 @@ wrong. However, since all communications between Docker clients and servers are done over HTTPS, it's a bit difficult to decrypt the traffic quickly even if you know the private key. What can we do instead? -One way would be to disable HTTPS by setting up an [insecure -Registry](https://docs.docker.com/registry/insecure/). This could introduce a +One way would be to disable HTTPS by setting up an +[insecure Registry](https://docs.docker.com/registry/insecure/). This could introduce a security hole and is only recommended for local testing. If you have a production system and can't or don't want to do this, there is another way: use mitmproxy, which stands for Man-in-the-Middle Proxy. diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index b58eeff48cf..e545fb3cd1b 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -78,7 +78,7 @@ among other things. Follow [these instructions](https://publicsuffix.org/submit/) to submit your GitLab Pages subdomain. For instance, if your domain is `example.io`, you should request that `example.io` is added to the Public Suffix List. GitLab.com -added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/infrastructure/-/issues/230). +added `gitlab.io` [in 2016](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/230). ### DNS configuration @@ -614,7 +614,7 @@ Follow the steps below to configure verbose logging of GitLab Pages daemon. Setting the `propagate_correlation_id` to true allows installations behind a reverse proxy to generate and set a correlation ID to requests sent to GitLab Pages. When a reverse proxy sets the header value `X-Request-ID`, the value propagates in the request chain. -Users [can find the correlation ID in the logs](../troubleshooting/tracing_correlation_id.md#identify-the-correlation-id-for-a-request). +Users [can find the correlation ID in the logs](../logs/tracing_correlation_id.md#identify-the-correlation-id-for-a-request). To enable the propagation of the correlation ID: @@ -933,8 +933,8 @@ The following settings are: | `connection` | Various connection options described below. | | NOTE: -If you want to stop using and disconnect the NFS server, you need to [explicitly disable -local storage](#disable-pages-local-storage), and it's only possible after upgrading to GitLab 13.11. +If you want to stop using and disconnect the NFS server, you need to +[explicitly disable local storage](#disable-pages-local-storage), and it's only possible after upgrading to GitLab 13.11. #### S3-compatible connection settings @@ -1319,25 +1319,6 @@ and in your Pages log shows this error: sudo gitlab-ctl restart gitlab-pages ``` -### 404 error after promoting a Geo secondary to a primary node - -Pages files are not among the -[supported data types](../geo/replication/datatypes.md#limitations-on-replicationverification) for replication in Geo. After a secondary node is promoted to a primary node, attempts to access a Pages site result in a `404 Not Found` error. - -It is possible to copy the subfolders and files in the [Pages path](#change-storage-path) -to the new primary node to resolve this. -For example, you can adapt the `rsync` strategy from the -[moving repositories documentation](../operations/moving_repositories.md). -Alternatively, run the CI pipelines of those projects that contain a `pages` job again. - -### 404 or 500 error when accessing GitLab Pages in a Geo setup - -Pages sites are only available on the primary Geo site, while the codebase of the project is available on all sites. - -If you try to access a Pages page on a secondary site, a 404 or 500 HTTP code is returned depending on the access control. - -Read more which [features don't support Geo replication/verification](../geo/replication/datatypes.md#limitations-on-replicationverification). - ### Failed to connect to the internal GitLab API If you see the following error: @@ -1495,7 +1476,7 @@ In GitLab 14.0-14.2 you can temporarily enable legacy storage and configuration To do that: -1. Please describe the issue you're seeing in the [migration feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). +1. Describe the issue you're seeing in the [migration feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). 1. Edit `/etc/gitlab/gitlab.rb`: diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index 4bb9fc56b6f..5ab508dbaca 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -14,8 +14,7 @@ This is the documentation for configuring a GitLab Pages when you have installed GitLab from source and not using the Omnibus packages. You are encouraged to read the [Omnibus documentation](index.md) as it provides -some invaluable information to the configuration of GitLab Pages. Please proceed -to read it before going forward with this guide. +invaluable information about the configuration of GitLab Pages. We also highly recommend that you use the Omnibus GitLab packages. We optimize them specifically for GitLab, and we take care of upgrading GitLab diff --git a/doc/administration/postgresql/database_load_balancing.md b/doc/administration/postgresql/database_load_balancing.md index b03abf9881c..f53b50192eb 100644 --- a/doc/administration/postgresql/database_load_balancing.md +++ b/doc/administration/postgresql/database_load_balancing.md @@ -180,7 +180,7 @@ gitlab_rails['db_load_balancing'] = { ## Logging The load balancer logs various events in -[`database_load_balancing.log`](../logs.md#database_load_balancinglog), such as +[`database_load_balancing.log`](../logs/index.md#database_load_balancinglog), such as - When a host is marked as offline - When a host comes back online diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index b14c1f64efe..8f27d41b75f 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -8,8 +8,7 @@ redirect_to: 'index.md' # Pseudonymizer (removed) **(ULTIMATE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in -> GitLab 14.7 and removed in 15.0. +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in GitLab 14.7 and removed in 15.0. WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/219952) in GitLab 14.7. diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 6de47c37957..a900da52aba 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -187,7 +187,7 @@ Checking GitLab ... Finished In some cases it is necessary to rebuild the `authorized_keys` file, for example, if after an upgrade you receive `Permission denied (publickey)` when pushing [via SSH](../../user/ssh.md) -and find `404 Key Not Found` errors in [the `gitlab-shell.log` file](../logs.md#gitlab-shelllog). +and find `404 Key Not Found` errors in [the `gitlab-shell.log` file](../logs/index.md#gitlab-shelllog). To rebuild `authorized_keys`, run: **Omnibus Installation** diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index f623d4e4526..f3a9845d129 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -17,11 +17,7 @@ GitLab provides Rake tasks relating to project import and export. For more infor You can query an import through the [Project import/export API](../../api/project_import_export.md#import-status). As described in the API documentation, the query may return an import error or exceptions. -## Import large projects - -If you have a larger project, consider using a Rake task, as described in our [developer documentation](../../development/import_project.md#importing-via-a-rake-task). - -## Import/export tasks +## Import/export Rake tasks The GitLab import/export version can be checked by using the following command: @@ -57,3 +53,38 @@ Note the following: - The exports are stored in a temporary directory and are deleted every 24 hours by a specific worker. + +### Import large projects using a Rake task + +If you have a larger project, consider using a Rake task as described in our [developer documentation](../../development/import_project.md#importing-via-a-rake-task). + +### Export using a Rake task + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25598) in GitLab 12.9. + +You can use a Rake task to export large project. + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `username` | string | yes | User name | +| `namespace_path` | string | yes | Namespace path | +| `project_path` | string | yes | Project name | +| `archive_path` | string | yes | Path to file to store the export project tarball | + +```shell +gitlab-rake "gitlab:import_export:export[username, namespace_path, project_path, archive_path]" +``` + +## Troubleshooting + +If you are having trouble with import/export, you can enable debug mode using the same Rake task: + +```shell +# Import +IMPORT_DEBUG=true gitlab-rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file_to_import.tar.gz]" + +# Export +EXPORT_DEBUG=true gitlab-rake "gitlab:import_export:export[root, group/subgroup, projectnametoexport, /tmp/export_file.tar.gz]" +``` diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index 9b1a456835a..c4b83b66738 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -16,14 +16,14 @@ In Redis lingo, `primary` is called `master`. In this document, `primary` is use instead of `master`, except the settings where `master` is required. Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -67,7 +67,7 @@ When a **Primary** fails to respond, it's the application's responsibility for a new **Primary**). To get a better understanding on how to correctly set up Sentinel, please read -the [Redis Sentinel documentation](https://redis.io/topics/sentinel) first, as +the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation first, as failing to configure it correctly can lead to data loss or can bring your whole cluster down, invalidating the failover effort. @@ -343,8 +343,8 @@ NOTE: If you are using an external Redis Sentinel instance, be sure to exclude the `requirepass` parameter from the Sentinel configuration. This parameter causes clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). +Authentication required.`. +[Redis Sentinel 3.2.x does not support password authentication](https://github.com/antirez/redis/issues/3279). Now that the Redis servers are all set up, let's configure the Sentinel servers. diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index 998455e5621..d624fe28f80 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -148,7 +148,7 @@ starting with `sentinel` prefix. Assuming that the Redis Sentinel is installed on the same instance as Redis primary with IP `10.0.0.1` (some settings might overlap with the primary): -1. [Install Redis Sentinel](https://redis.io/topics/sentinel). +1. [Install Redis Sentinel](https://redis.io/docs/manual/sentinel/). 1. Edit `/etc/redis/sentinel.conf`: ```conf diff --git a/doc/administration/redis/troubleshooting.md b/doc/administration/redis/troubleshooting.md index ca52fe0a29a..f9e5390c227 100644 --- a/doc/administration/redis/troubleshooting.md +++ b/doc/administration/redis/troubleshooting.md @@ -167,4 +167,4 @@ production: port: 26379 # point to sentinel, not to redis port ``` -When in doubt, read the [Redis Sentinel documentation](https://redis.io/topics/sentinel). +When in doubt, read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index a64828be4be..fcc32100bc8 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -693,7 +693,8 @@ SSH in to any of the Patroni nodes on the **primary site**: ``` If the 'State' column for any node doesn't say "running", check the -[Troubleshooting section](troubleshooting.md) before proceeding. +[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +before proceeding. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -800,14 +801,14 @@ The following IPs will be used as an example: ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -2062,10 +2063,8 @@ On each node perform the following: ``` 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: - - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the @@ -2086,7 +2085,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 5b1e8bfc16b..51d3e3d39a6 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -696,7 +696,8 @@ SSH in to any of the Patroni nodes on the **primary site**: ``` If the 'State' column for any node doesn't say "running", check the -[Troubleshooting section](troubleshooting.md) before proceeding. +[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +before proceeding. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -803,14 +804,14 @@ The following IPs will be used as an example: ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -2067,10 +2068,8 @@ On each node perform the following: ``` 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: - - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the @@ -2091,7 +2090,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index df84bf0402d..8ca58da55ca 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -512,10 +512,8 @@ Updates to example must be made at: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Confirm that Gitaly can perform callbacks to the internal API: - - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. ### Gitaly TLS support @@ -658,10 +656,12 @@ On each node perform the following: # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' gitlab_workhorse['prometheus_listen_addr'] = '0.0.0.0:9229' + puma['listen'] = '0.0.0.0' sidekiq['listen_address'] = "0.0.0.0" - # Set number of Sidekiq threads per queue process to the recommend number of 10 + + # Configure Sidekiq with 2 workers and 10 max concurrency sidekiq['max_concurrency'] = 10 - puma['listen'] = '0.0.0.0' + sidekiq['queue_groups'] = ['*'] * 2 # Add the monitoring node's IP address to the monitoring whitelist and allow it to # scrape the NGINX metrics. Replace placeholder `monitoring.gitlab.example.com` with @@ -766,7 +766,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1011,8 +1011,8 @@ future with further specific cloud provider details. | Service | Nodes | Configuration | GCP | AWS | Min Allocatable CPUs and Memory | |-----------------------------------------------|-------|------------------------|-----------------|--------------|---------------------------------| | Webservice | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | 23.7 vCPU, 16.9 GB memory | -| Sidekiq | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 3.9 vCPU, 11.8 GB memory | -| Supporting services such as NGINX, Prometheus | 2 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | 1.9 vCPU, 5.5 GB memory | +| Sidekiq | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | 3.9 vCPU, 11.8 GB memory | +| Supporting services such as NGINX, Prometheus | 2 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | 1.9 vCPU, 5.5 GB memory | - For this setup, we **recommend** and regularly [test](index.md#validation-and-test-results) [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) and [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/). Other Kubernetes services may also work, but your mileage may vary. @@ -1048,8 +1048,8 @@ card "Kubernetes via Helm Charts" as kubernetes { together { collections "**Webservice** x3" as gitlab #32CD32 - collections "**Sidekiq** x2" as sidekiq #ff8dd1 - card "**Supporting Services**" as support + card "**Sidekiq**" as sidekiq #ff8dd1 + collections "**Supporting Services** x2" as support } } @@ -1082,13 +1082,13 @@ documents how to apply the calculated configuration to the Helm Chart. #### Webservice Webservice pods typically need about 1 vCPU and 1.25 GB of memory _per worker_. -Each Webservice pod consumes roughly 2 vCPUs and 2.5 GB of memory using +Each Webservice pod consumes roughly 4 vCPUs and 5 GB of memory using the [recommended topology](#cluster-topology) because two worker processes are created by default and each pod has other small processes running. For 2,000 users we recommend a total Puma worker count of around 12. -With the [provided recommendations](#cluster-topology) this allows the deployment of up to 6 -Webservice pods with 2 workers per pod and 2 pods per node. Expand available resources using +With the [provided recommendations](#cluster-topology) this allows the deployment of up to 3 +Webservice pods with 4 workers per pod and 1 pod per node. Expand available resources using the ratio of 1 vCPU to 1.25 GB of memory _per each worker process_ for each additional Webservice pod. @@ -1099,7 +1099,7 @@ For further information on resource usage, see the [Webservice resources](https: Sidekiq pods should generally have 1 vCPU and 2 GB of memory. [The provided starting point](#cluster-topology) allows the deployment of up to -2 Sidekiq pods. Expand available resources using the 1 vCPU to 2GB memory +4 Sidekiq pods. Expand available resources using the 1 vCPU to 2 GB memory ratio for each additional pod. For further information on resource usage, see the [Sidekiq resources](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/#resources). diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 6a70739ca31..db81ae7ae3c 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -425,14 +425,14 @@ all nodes. ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -978,7 +978,8 @@ SSH in to any of the Patroni nodes on the **primary site**: ``` If the 'State' column for any node doesn't say "running", check the -[Troubleshooting section](troubleshooting.md) before proceeding. +[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +before proceeding. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2018,7 +2019,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 0d0e44e2364..535e483cb7d 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -703,7 +703,8 @@ SSH in to any of the Patroni nodes on the **primary site**: ``` If the 'State' column for any node doesn't say "running", check the -[Troubleshooting section](troubleshooting.md) before proceeding. +[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +before proceeding. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -810,14 +811,14 @@ The following IPs will be used as an example: ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -2083,10 +2084,8 @@ On each node perform the following: ``` 1. Optionally, from the Gitaly servers, confirm that Gitaly can perform callbacks to the internal API: - - ```shell - sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml - ``` + - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. + - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the @@ -2107,7 +2106,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index ef2e48baa33..e19440e7660 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -423,14 +423,14 @@ all nodes. ## Configure Redis Using [Redis](https://redis.io/) in scalable environment is possible using a **Primary** x **Replica** -topology with a [Redis Sentinel](https://redis.io/topics/sentinel) service to watch and automatically +topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. Redis requires authentication if used with Sentinel. See -[Redis Security](https://redis.io/topics/security) documentation for more +[Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight firewall rules to secure your Redis service. -You are highly encouraged to read the [Redis Sentinel](https://redis.io/topics/sentinel) documentation +You are highly encouraged to read the [Redis Sentinel](https://redis.io/docs/manual/sentinel/) documentation before configuring Redis with GitLab to fully understand the topology and architecture. @@ -975,7 +975,8 @@ SSH in to any of the Patroni nodes on the **primary site**: ``` If the 'State' column for any node doesn't say "running", check the -[Troubleshooting section](troubleshooting.md) before proceeding. +[PostgreSQL replication and failover troubleshooting section](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) +before proceeding. <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> @@ -2018,7 +2019,7 @@ the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#en failing to connect to PostgreSQL, it may be that your PgBouncer node's IP address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index e996acb1157..7be12e12386 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -240,7 +240,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=b51f178f4403b69a63f6eb33ea425f82de3bf249">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/1adf30bef7e34ceba9efa97c4470417b">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=1adf30bef7e34ceba9efa97c4470417b">Calculated cost</a></td> </tr> <tr> <th scope="row">2k</th> @@ -248,7 +248,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=dce36b5cb6ab25211f74e47233d77f58fefb54e2">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/72764902f3854f798407fb03c3de4b6f">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=72764902f3854f798407fb03c3de4b6f">Calculated cost</a></td> </tr> <tr> <th scope="row">3k</th> @@ -256,7 +256,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=b1c5b4e32e990eaeb035a148255132bd28988760">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/0dbfc575051943b9970e5d8ace03680d">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=0dbfc575051943b9970e5d8ace03680d">Calculated cost</a></td> </tr> <tr> <th scope="row">5k</th> @@ -264,7 +264,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=2bf1af883096e6f4c6efddb4f3c35febead7fec2">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/8f618711ffec4b039f1581871ca6a7c9">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=8f618711ffec4b039f1581871ca6a7c9">Calculated cost</a></td> </tr> <tr> <th scope="row">10k</th> @@ -272,7 +272,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=1d374df13c0f2088d332ab0134f5b1d0f717259e">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/de3da8286dda4d4db1362932bc75410b">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=de3da8286dda4d4db1362932bc75410b">Calculated cost</a></td> </tr> <tr> <th scope="row">25k</th> @@ -280,7 +280,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=46fe6a6e9256d9b7779fae59fbbfa7e836942b7d">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/69724ebd82914a60857da6a3ace05a64">Calculate cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=69724ebd82914a60857da6a3ace05a64">Calculate cost</a></td> </tr> <tr> <th scope="row">50k</th> @@ -288,7 +288,7 @@ The following table details the cost to run the different reference architecture <td></td> <td><a href="https://calculator.aws/#/estimate?id=e15926b1a3c7139e4faf390a3875ff807d2ab91c">Calculated cost</a></td> <td></td> - <td><a href="https://azure.com/e/3f973040ebc14023933d35f576c89846">Calculated cost</a></td> + <td><a href="https://azure.microsoft.com/en-us/pricing/calculator/?shared-estimate=3f973040ebc14023933d35f576c89846">Calculated cost</a></td> </tr> </table> diff --git a/doc/administration/reference_architectures/troubleshooting.md b/doc/administration/reference_architectures/troubleshooting.md index 6e0aaacd598..a9bf035a528 100644 --- a/doc/administration/reference_architectures/troubleshooting.md +++ b/doc/administration/reference_architectures/troubleshooting.md @@ -1,352 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../configure.md' +remove_date: '2022-10-24' --- -# Troubleshooting a reference architecture setup **(FREE SELF)** +This document was moved to [another location](../configure.md). -This page serves as the troubleshooting documentation if you followed one of -the [reference architectures](index.md#reference-architectures). - -## Troubleshooting object storage - -### S3 API compatibility issues - -Not all S3 providers [are fully compatible](../../raketasks/backup_gitlab.md#other-s3-providers) -with the Fog library that GitLab uses. Symptoms include: - -```plaintext -411 Length Required -``` - -### GitLab Pages can use object storage - -If you intend to use [GitLab Pages](../../user/project/pages/index.md), you can -[configure object storage](../pages/index.md#using-object-storage). -NFS is still available if you prefer. - -The [GitLab Pages Helm chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-pages/) is also available -for Kubernetes deployments. - -### Incremental logging is required for CI to use object storage - -If you configure GitLab to use object storage for CI logs and artifacts, -[you must also enable incremental logging](../job_logs.md#incremental-logging-architecture). - -### Proxy Download - -A number of the use cases for object storage allow client traffic to be redirected to the -object storage back end, like when Git clients request large files via LFS or when -downloading CI artifacts and logs. - -When the files are stored on local block storage or NFS, GitLab has to act as a proxy. -With object storage, the default behavior is for GitLab to redirect to the object -storage device rather than proxy the request. - -The `proxy_download` setting controls this behavior: the default is generally `false`. -Verify this in the documentation for each use case. Set it to `true` to make -GitLab proxy the files rather than redirect. - -When not proxying files, GitLab returns an -[HTTP 302 redirect with a pre-signed, time-limited object storage URL](https://gitlab.com/gitlab-org/gitlab/-/issues/32117#note_218532298). -This can result in some of the following problems: - -- If GitLab is using non-secure HTTP to access the object storage, clients may generate -`https->http` downgrade errors and refuse to process the redirect. The solution to this -is for GitLab to use HTTPS. LFS, for example, generates this error: - - ```plaintext - LFS: lfsapi/client: refusing insecure redirect, https->http - ``` - -- Clients must trust the certificate authority that issued the object storage -certificate, or may return common TLS errors such as: - - ```plaintext - x509: certificate signed by unknown authority - ``` - -- Clients need network access to the object storage. Errors that might result -if this access is not in place include: - - ```plaintext - Received status code 403 from server: Forbidden - ``` - -### ETag mismatch - -Using the default GitLab settings, some object storage back-ends such as -[MinIO](https://gitlab.com/gitlab-org/gitlab/-/issues/23188) -and [Alibaba](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564) -might generate `ETag mismatch` errors. - -When using GitLab direct upload, the -[workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658) -is to use the `--compat` parameter on the server. - -We are working on a fix to GitLab component Workhorse, and also -a workaround, in the mean time, to -[allow ETag verification to be disabled](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18175). - -## Troubleshooting Redis - -There are a lot of moving parts that needs to be taken care carefully -in order for the HA setup to work as expected. - -Before proceeding with the troubleshooting below, check your firewall rules: - -- Redis machines - - Accept TCP connection in `6379` - - Connect to the other Redis machines via TCP in `6379` -- Sentinel machines - - Accept TCP connection in `26379` - - Connect to other Sentinel machines via TCP in `26379` - - Connect to the Redis machines via TCP in `6379` - -### Troubleshooting Redis replication - -You can check if everything is correct by connecting to each server using -`redis-cli` application, and sending the `info replication` command as below. - -```shell -/opt/gitlab/embedded/bin/redis-cli -h <redis-host-or-ip> -a '<redis-password>' info replication -``` - -When connected to a `Primary` Redis, you see the number of connected -`replicas`, and a list of each with connection details: - -```plaintext -# Replication -role:master -connected_replicas:1 -replica0:ip=10.133.5.21,port=6379,state=online,offset=208037514,lag=1 -master_repl_offset:208037658 -repl_backlog_active:1 -repl_backlog_size:1048576 -repl_backlog_first_byte_offset:206989083 -repl_backlog_histlen:1048576 -``` - -When it's a `replica`, you see details of the primary connection and if -its `up` or `down`: - -```plaintext -# Replication -role:replica -master_host:10.133.1.58 -master_port:6379 -master_link_status:up -master_last_io_seconds_ago:1 -master_sync_in_progress:0 -replica_repl_offset:208096498 -replica_priority:100 -replica_read_only:1 -connected_replicas:0 -master_repl_offset:0 -repl_backlog_active:0 -repl_backlog_size:1048576 -repl_backlog_first_byte_offset:0 -repl_backlog_histlen:0 -``` - -### Troubleshooting Sentinel - -If you get an error like: `Redis::CannotConnectError: No sentinels available.`, -there may be something wrong with your configuration files or it can be related -to [this issue](https://github.com/redis/redis-rb/issues/531). - -You must make sure you are defining the same value in `redis['master_name']` -and `redis['master_password']` as you defined for your sentinel node. - -The way the Redis connector `redis-rb` works with sentinel is a bit -non-intuitive. We try to hide the complexity in omnibus, but it still requires -a few extra configurations. - ---- - -To make sure your configuration is correct: - -1. SSH into your GitLab application server -1. Enter the Rails console: - - ```shell - # For Omnibus installations - sudo gitlab-rails console - - # For source installations - sudo -u git rails console -e production - ``` - -1. Run in the console: - - ```ruby - redis = Redis.new(Gitlab::Redis::SharedState.params) - redis.info - ``` - - Keep this screen open and try to simulate a failover below. - -1. To simulate a failover on primary Redis, SSH into the Redis server and run: - - ```shell - # port must match your primary redis port, and the sleep time must be a few seconds bigger than defined one - redis-cli -h localhost -p 6379 DEBUG sleep 20 - ``` - -1. Then back in the Rails console from the first step, run: - - ```ruby - redis.info - ``` - - You should see a different port after a few seconds delay - (the failover/reconnect time). - -## Troubleshooting Gitaly - -For troubleshooting information, see Gitaly and Gitaly Cluster -[troubleshooting information](../gitaly/troubleshooting.md). - -## Troubleshooting the GitLab Rails application - -- `mount: wrong fs type, bad option, bad superblock on` - -You have not installed the necessary NFS client utilities. See step 1 above. - -- `mount: mount point /var/opt/gitlab/... does not exist` - -This particular directory does not exist on the NFS server. Ensure -the share is exported and exists on the NFS server and try to remount. - -## Troubleshooting Monitoring - -If the monitoring node is not receiving any data, check that the exporters are -capturing data. - -```shell -curl "http[s]://localhost:<EXPORTER LISTENING PORT>/metric" -``` - -or - -```shell -curl "http[s]://localhost:<EXPORTER LISTENING PORT>/-/metric" -``` - -## Troubleshooting PgBouncer - -In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs: - -```shell -sudo gitlab-ctl tail pgbouncer -``` - -Additionally, you can check the output from `show databases` in the [administrative console](#pgbouncer-administrative-console). In the output, you would expect to see values in the `host` field for the `gitlabhq_production` database. Additionally, `current_connections` should be greater than 1. - -### PgBouncer administrative console - -As part of Omnibus GitLab, the `gitlab-ctl pgb-console` command is provided to automatically connect to the PgBouncer administrative console. See the [PgBouncer documentation](https://www.pgbouncer.org/usage.html#admin-console) for detailed instructions on how to interact with the console. - -To start a session: - -```shell -sudo gitlab-ctl pgb-console -``` - -The password you are prompted for is the `pgbouncer_user_password` - -To get some basic information about the instance, run - -```shell -pgbouncer=# show databases; show clients; show servers; - name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections ----------------------+-----------+------+---------------------+------------+-----------+--------------+-----------+-----------------+--------------------- - gitlabhq_production | 127.0.0.1 | 5432 | gitlabhq_production | | 100 | 5 | | 0 | 1 - pgbouncer | | 6432 | pgbouncer | pgbouncer | 2 | 0 | statement | 0 | 0 -(2 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link -| remote_pid | tls -------+-----------+---------------------+--------+-----------+-------+------------+------------+---------------------+---------------------+-----------+------ -+------------+----- - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44590 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12444c0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44592 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x12447c0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44594 | 127.0.0.1 | 6432 | 2018-04-24 22:13:10 | 2018-04-24 22:17:10 | 0x1244940 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44706 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:16:31 | 0x1244ac0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44708 | 127.0.0.1 | 6432 | 2018-04-24 22:14:22 | 2018-04-24 22:15:15 | 0x1244c40 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44794 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:15:15 | 0x1244dc0 | -| 0 | - C | gitlab | gitlabhq_production | active | 127.0.0.1 | 44798 | 127.0.0.1 | 6432 | 2018-04-24 22:15:15 | 2018-04-24 22:16:31 | 0x1244f40 | -| 0 | - C | pgbouncer | pgbouncer | active | 127.0.0.1 | 44660 | 127.0.0.1 | 6432 | 2018-04-24 22:13:51 | 2018-04-24 22:17:12 | 0x1244640 | -| 0 | -(8 rows) - - type | user | database | state | addr | port | local_addr | local_port | connect_time | request_time | ptr | link | rem -ote_pid | tls -------+--------+---------------------+-------+-----------+------+------------+------------+---------------------+---------------------+-----------+------+---- ---------+----- - S | gitlab | gitlabhq_production | idle | 127.0.0.1 | 5432 | 127.0.0.1 | 35646 | 2018-04-24 22:15:15 | 2018-04-24 22:17:10 | 0x124dca0 | | - 19980 | -(1 row) -``` - -### Message: `LOG: invalid CIDR mask in address` - -See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-cidr-mask-in-address). - -### Message: `LOG: invalid IP mask "md5": Name or service not known` - -See the suggested fix [in Geo documentation](../geo/replication/troubleshooting.md#message-log--invalid-ip-mask-md5-name-or-service-not-known). - -## Troubleshooting PostgreSQL with Patroni - -In case you are experiencing any issues connecting through PgBouncer, the first place to check is always the logs for PostgreSQL (which is run through Patroni): - -```shell -sudo gitlab-ctl tail patroni -``` - -### Consul and PostgreSQL with Patroni changes not taking effect - -Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it does not restart the services. However, not all changes can be activated by reloading. - -To restart either service, run `gitlab-ctl restart consul` or `gitlab-ctl restart patroni` respectively. - -For PostgreSQL with Patroni, to prevent the primary node from being failed over automatically, it's safest to stop all secondaries first, then restart the primary and finally restart the secondaries again. - -On the Consul server nodes, it is important to restart the Consul service in a controlled fashion. Read our [Consul documentation](../consul.md#restart-consul) for instructions on how to restart the service. - -### PgBouncer error `ERROR: pgbouncer cannot connect to server` - -You may get this error when running `gitlab-rake gitlab:db:configure` or you -may see the error in the PgBouncer log file. - -```plaintext -PG::ConnectionBad: ERROR: pgbouncer cannot connect to server -``` - -The problem may be that your PgBouncer node's IP address is not included in the -`trust_auth_cidr_addresses` setting in `/etc/gitlab/gitlab.rb` on the database nodes. - -You can confirm that this is the issue by checking the PostgreSQL log on the primary -database node. If you see the following error then `trust_auth_cidr_addresses` -is the problem. - -```plaintext -2018-03-29_13:59:12.11776 FATAL: no pg_hba.conf entry for host "123.123.123.123", user "pgbouncer", database "gitlabhq_production", SSL off -``` - -To fix the problem, add the IP address to `/etc/gitlab/gitlab.rb`. - -```ruby -postgresql['trust_auth_cidr_addresses'] = %w(123.123.123.123/32 <other_cidrs>) -``` - -[Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +<!-- This redirect file can be deleted after <2022-10-24>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/administration/repository_checks.md b/doc/administration/repository_checks.md index 8cfefdb9b56..a97c8611239 100644 --- a/doc/administration/repository_checks.md +++ b/doc/administration/repository_checks.md @@ -7,8 +7,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Repository checks **(FREE SELF)** You can use [`git fsck`](https://git-scm.com/docs/git-fsck) to verify the integrity of all data -committed to a repository. GitLab administrators can trigger this check for a project using the -GitLab UI: +committed to a repository. GitLab administrators can: + +- Manually trigger this check for a project, using the GitLab UI. +- Schedule this check to run automatically for all projects. +- Run this check from the command line. +- Run a [Rake task](raketasks/check.md#repository-integrity) for checking Git repositories, which can be used to run + `git fsck` against all repositories and generate repository checksums, as a way to compare repositories on different + servers. + +## Check a project's repository using GitLab UI + +To check a project's repository using GitLab UI: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Projects**. @@ -18,9 +28,7 @@ GitLab UI: The checks run asynchronously so it may take a few minutes before the check result is visible on the project page in the Admin Area. If the checks fail, see [what to do](#what-to-do-if-a-check-failed). -This setting is off by default, because it can cause many false alarms. - -## Enable periodic checks +## Enable repository checks for all projects Instead of checking repositories manually, GitLab can be configured to run the checks periodically: @@ -45,13 +53,31 @@ the start of Sunday. Repositories with known check failures can be found at `/admin/projects?last_repository_check_failed=1`. +## Run a check using the command line + +You can run [`git fsck`](https://git-scm.com/docs/git-fsck) using the command line on repositories on +[Gitaly servers](gitaly/index.md). To locate the repositories: + +1. Go to the storage location for repositories: + - For Omnibus GitLab installations, repositories are stored in the `/var/opt/gitlab/git-data/repositories` directory + by default. + - For GitLab Helm chart installations, repositories are stored in the `/home/git/repositories` directory inside the + Gitaly pod by default. +1. [Identify the subdirectory that contains the repository](repository_storage_types.md#from-project-name-to-hashed-path) + that you need to check. +1. Run the check. For example: + + ```shell + sudo /opt/gitlab/embedded/bin/git -C /var/opt/gitlab/git-data/repositories/@hashed/0b/91/0b91...f9.git fsck + ``` + ## What to do if a check failed -If a repository check fails, locate the error in the [`repocheck.log` file](logs.md#repochecklog) on -disk at: +If a repository check fails, locate the error in the [`repocheck.log` file](logs/index.md#repochecklog) on disk at: - `/var/log/gitlab/gitlab-rails` for Omnibus GitLab installations. - `/home/git/gitlab/log` for installations from source. +- `/var/log/gitlab` in the Sidekiq pod for GitLab Helm chart installations. If periodic repository checks cause false alarms, you can clear all repository check states: @@ -59,23 +85,3 @@ If periodic repository checks cause false alarms, you can clear all repository c 1. On the left sidebar, select **Settings > Repository** (`/admin/application_settings/repository`). 1. Expand the **Repository maintenance** section. 1. Select **Clear all repository checks**. - -## Run a check using the command line - -You can run [`git fsck`](https://git-scm.com/docs/git-fsck) using the command line on repositories -on [Gitaly servers](gitaly/index.md). To locate the repositories: - -1. Go to the storage location for repositories. For Omnibus GitLab installations, repositories are - stored by default in the `/var/opt/gitlab/git-data/repositories` directory. -1. [Identify the subdirectory that contains the repository](repository_storage_types.md#from-project-name-to-hashed-path) - that you need to check. - -To run a check (for example): - -```shell -sudo /opt/gitlab/embedded/bin/git -C /var/opt/gitlab/git-data/repositories/@hashed/0b/91/0b91...f9.git fsck -``` - -You can also run [Rake tasks](raketasks/check.md#repository-integrity) for checking Git -repositories, which can be used to run `git fsck` against all repositories and generate repository -checksums, as a way to compare repositories on different servers. diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 6ac031ea6bd..6625039504a 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -102,8 +102,8 @@ depend on those files. ## Installations from source -If you have followed the official installation guide to [install GitLab from -source](../install/installation.md), run the following command to restart GitLab: +If you have followed the official installation guide to +[install GitLab from source](../install/installation.md), run the following command to restart GitLab: ```shell # For systems running systemd diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md index fc24c764330..01f83f98607 100644 --- a/doc/administration/sidekiq.md +++ b/doc/administration/sidekiq.md @@ -1,385 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'sidekiq/index.md' +remove_date: '2022-11-11' --- -# Configure an external Sidekiq instance **(FREE SELF)** +This document was moved to [another location](sidekiq/index.md). -You can configure an external Sidekiq instance by using the Sidekiq that's bundled in the GitLab package. Sidekiq requires connection to the Redis, -PostgreSQL, and Gitaly instances. - -## Configure TCP access for PostgreSQL, Gitaly, and Redis - -By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. To change this: - -1. Edit the `/etc/gitlab/gitlab.rb` file on your GitLab instance, and add the following: - - ```ruby - - ## PostgreSQL - - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - postgresql['listen_address'] = '0.0.0.0' - postgresql['port'] = 5432 - - # Add the Sidekiq nodes to PostgreSQL's trusted addresses. - # In the following example, 10.10.1.30/32 is the private IP - # of the Sidekiq server. - postgresql['md5_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32) - postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32) - - ## Gitaly - - # Make Gitaly accept connections on all network interfaces - gitaly['listen_addr'] = "0.0.0.0:8075" - ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network - ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token - gitaly['auth_token'] = 'abc123secret' - praefect['auth_token'] = 'abc123secret' - gitlab_rails['gitaly_token'] = 'abc123secret' - - ## Redis configuration - - redis['bind'] = '0.0.0.0' - redis['port'] = 6379 - # Password to Authenticate Redis - redis['password'] = 'redis-password-goes-here' - gitlab_rails['redis_password'] = 'redis-password-goes-here' - - gitlab_rails['auto_migrate'] = false - ``` - -1. Run `reconfigure`: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -1. Restart the `PostgreSQL` server: - - ```shell - sudo gitlab-ctl restart postgresql - ``` - -1. After the restart, set `auto_migrate` to `true` or comment to use the default settings: - - ```ruby - gitlab_rails['auto_migrate'] = true - ``` - -1. Run `reconfigure` again: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Set up Sidekiq instance - -1. SSH into the Sidekiq server. - -1. Confirm that you can access the PostgreSQL, Gitaly, and Redis ports: - - ```shell - telnet <GitLab host> 5432 # PostgreSQL - telnet <GitLab host> 8075 # Gitaly - telnet <GitLab host> 6379 # Redis - ``` - -1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package - using steps 1 and 2. **Do not complete any other steps.** - -1. Copy the `/etc/gitlab/gitlab.rb` file from the GitLab instance and add the following settings. Make sure - to replace them with your values: - -<!-- -Updates to example must be made at: -- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md -- all reference architecture pages ---> - - ```ruby - ######################################## - ##### Services Disabled ### - ######################################## - # - # When running GitLab on just one server, you have a single `gitlab.rb` - # to enable all services you want to run. - # When running GitLab on N servers, you have N `gitlab.rb` files. - # Enable only the services you want to run on each - # specific server, while disabling all others. - # - gitaly['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - nginx['enable'] = false - puma['enable'] = false - gitlab_workhorse['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - grafana['enable'] = false - gitlab_exporter['enable'] = false - gitlab_kas['enable'] = false - - ## - ## To maintain uniformity of links across nodes, the - ## `external_url` on the Sidekiq server should point to the external URL that users - ## use to access GitLab. This can be either: - ## - ## - The `external_url` set on your application server. - ## - The URL of a external load balancer, which routes traffic to the GitLab application server. - ## - external_url 'https://gitlab.example.com' - - # Configure the gitlab-shell API callback URL. Without this, `git push` will - # fail. This can be your 'front door' GitLab URL or an internal load - # balancer. - gitlab_rails['internal_api_url'] = 'GITLAB_URL' - gitlab_shell['secret_token'] = 'SHELL_TOKEN' - - ######################################## - #### Redis ### - ######################################## - - ## Must be the same in every sentinel node. - redis['master_name'] = 'gitlab-redis' # Required if you have setup redis cluster - ## The same password for Redis authentication you set up for the master node. - redis['master_password'] = '<redis_master_password>' - - ### If redis is running on the main Gitlab instance and you have opened the TCP port as above add the following - gitlab_rails['redis_host'] = '<gitlab_host>' - gitlab_rails['redis_port'] = 6379 - - ####################################### - ### Gitaly ### - ####################################### - - ## Replace <gitaly_token> with the one you set up, see - ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token - git_data_dirs({ - "default" => { - "gitaly_address" => "tcp://<gitlab_host>:8075", - "gitaly_token" => "<gitaly_token>" - } - }) - - ####################################### - ### Postgres ### - ####################################### - - # Replace <database_host> and <database_password> - gitlab_rails['db_host'] = '<database_host>' - gitlab_rails['db_port'] = '5432' - gitlab_rails['db_password'] = '<database_password>' - ## Prevent database migrations from running on upgrade automatically - gitlab_rails['auto_migrate'] = false - - ####################################### - ### Sidekiq configuration ### - ####################################### - sidekiq['enable'] = true - sidekiq['listen_address'] = "0.0.0.0" - - ## Set number of Sidekiq queue processes to the same number as available CPUs - sidekiq['queue_groups'] = ['*'] * 4 - - ## Set number of Sidekiq threads per queue process to the recommend number of 10 - sidekiq['max_concurrency'] = 10 - ``` - -1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the GitLab instance and replace the file in the Sidekiq instance. - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -1. Restart the Sidekiq instance after completing the process and finishing the database migrations. - -## Configure multiple Sidekiq nodes with shared storage - -If you run multiple Sidekiq nodes with a shared file storage, such as NFS, you must -specify the UIDs and GIDs to ensure they match between servers. Specifying the UIDs -and GIDs prevents permissions issues in the file system. This advice is similar to the -[advice for Geo setups](geo/replication/multiple_servers.md#step-4-configure-the-frontend-application-nodes-on-the-geo-secondary-site). - -To set up multiple Sidekiq nodes: - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Configure the Container Registry when using an external Sidekiq - -If you're using the Container Registry and it's running on a different -node than Sidekiq, follow the steps below. - -1. Edit `/etc/gitlab/gitlab.rb`, and configure the registry URL: - - ```ruby - registry_external_url 'https://registry.example.com' - gitlab_rails['registry_api_url'] = "https://registry.example.com" - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -1. In the instance where Container Registry is hosted, copy the `registry.key` - file to the Sidekiq node. - -## Configure the Sidekiq metrics server - -If you want to collect Sidekiq metrics, enable the Sidekiq metrics server. -To make metrics available from `localhost:8082/metrics`: - -To configure the metrics server: - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - sidekiq['metrics_enabled'] = true - sidekiq['listen_address'] = "localhost" - sidekiq['listen_port'] = "8082" - - # Optionally log all the metrics server logs to log/sidekiq_exporter.log - sidekiq['exporter_log_enabled'] = true - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -### Enable HTTPS - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364771) in GitLab 15.2. - -To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings: - -1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines: - - ```ruby - sidekiq['exporter_tls_enabled'] = true - sidekiq['exporter_tls_cert_path'] = "/path/to/certificate.pem" - sidekiq['exporter_tls_key_path'] = "/path/to/private-key.pem" - ``` - -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) - for the changes to take effect. - -When TLS is enabled, the same `port` and `address` are used as described above. -The metrics server cannot serve both HTTP and HTTPS at the same time. - -## Configure health checks - -If you use health check probes to observe Sidekiq, enable the Sidekiq health check server. -To make health checks available from `localhost:8092`: - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - sidekiq['health_checks_enabled'] = true - sidekiq['health_checks_listen_address'] = "localhost" - sidekiq['health_checks_listen_port'] = "8092" - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -For more information about health checks, see the [Sidekiq health check page](sidekiq_health_check.md). - -## Configure LDAP and user or group synchronization - -If you use LDAP for user and group management, you must add the LDAP configuration to your Sidekiq node as well as the LDAP -synchronization worker. If the LDAP configuration and LDAP synchronization worker are not applied to your Sidekiq node, -users and groups are not automatically synchronized. - -For more information about configuring LDAP for GitLab, see: - -- [GitLab LDAP configuration documentation](auth/ldap/index.md#configure-ldap) -- [LDAP synchronization documentation](auth/ldap/ldap_synchronization.md#adjust-ldap-user-sync-schedule) - -To enable LDAP with the synchronization worker for Sidekiq: - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['ldap_enabled'] = true - gitlab_rails['prevent_ldap_sign_in'] = false - gitlab_rails['ldap_servers'] = { - 'main' => { - 'label' => 'LDAP', - 'host' => 'ldap.mydomain.com', - 'port' => 389, - 'uid' => 'sAMAccountName', - 'encryption' => 'simple_tls', - 'verify_certificates' => true, - 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', - 'password' => '_the_password_of_the_bind_user', - 'tls_options' => { - 'ca_file' => '', - 'ssl_version' => '', - 'ciphers' => '', - 'cert' => '', - 'key' => '' - }, - 'timeout' => 10, - 'active_directory' => true, - 'allow_username_or_email_login' => false, - 'block_auto_created_users' => false, - 'base' => 'dc=example,dc=com', - 'user_filter' => '', - 'attributes' => { - 'username' => ['uid', 'userid', 'sAMAccountName'], - 'email' => ['mail', 'email', 'userPrincipalName'], - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' - }, - 'lowercase_usernames' => false, - - # Enterprise Edition only - # https://docs.gitlab.com/ee/administration/auth/ldap/ldap_synchronization.html - 'group_base' => '', - 'admin_group' => '', - 'external_groups' => [], - 'sync_ssh_keys' => false - } - } - gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -## Related topics - -- [Extra Sidekiq processes](operations/extra_sidekiq_processes.md) -- [Extra Sidekiq routing](operations/extra_sidekiq_routing.md) -- [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) -- [Sidekiq health checks](sidekiq_health_check.md) +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/sidekiq/extra_sidekiq_processes.md b/doc/administration/sidekiq/extra_sidekiq_processes.md new file mode 100644 index 00000000000..1cd3771c94d --- /dev/null +++ b/doc/administration/sidekiq/extra_sidekiq_processes.md @@ -0,0 +1,362 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Run multiple Sidekiq processes **(FREE SELF)** + +GitLab allows you to start multiple Sidekiq processes. +These processes can be used to consume a dedicated set +of queues. This can be used to ensure certain queues always have dedicated +workers, no matter the number of jobs to be processed. + +NOTE: +The information in this page applies only to Omnibus GitLab. + +## Available Sidekiq queues + +For a list of the existing Sidekiq queues, check the following files: + +- [Queues for both GitLab Community and Enterprise Editions](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) +- [Queues for GitLab Enterprise Editions only](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml) + +Each entry in the above files represents a queue on which Sidekiq processes +can be started. + +## Start multiple processes + +> - [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4006) in GitLab 12.10, starting multiple processes with Sidekiq cluster. +> - [Sidekiq cluster moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. +> - [Sidekiq cluster became default](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4140) in GitLab 13.0. + +When starting multiple processes, the number of processes should +equal (and **not** exceed) the number of CPU cores you want to +dedicate to Sidekiq. Each Sidekiq process can use only 1 CPU +core, subject to the available workload and concurrency settings. + +To start multiple processes: + +1. Using the `sidekiq['queue_groups']` array setting, specify how many processes to + create using `sidekiq-cluster` and which queue they should handle. + Each item in the array equates to one additional Sidekiq + process, and values in each item determine the queues it works on. + + For example, the following setting creates three Sidekiq processes, one to run on + `elastic_commit_indexer`, one to run on `mailers`, and one process running on all queues: + + ```ruby + sidekiq['queue_groups'] = [ + "elastic_commit_indexer", + "mailers", + "*" + ] + ``` + + To have an additional Sidekiq process handle multiple queues, add multiple + queue names to its item delimited by commas. For example: + + ```ruby + sidekiq['queue_groups'] = [ + "elastic_commit_indexer, elastic_association_indexer", + "mailers", + "*" + ] + ``` + + [In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and + later, the special queue name `*` means all queues. This starts two + processes, each handling all queues: + + ```ruby + sidekiq['queue_groups'] = [ + "*", + "*" + ] + ``` + + `*` cannot be combined with concrete queue names - `*, mailers` + just handles the `mailers` queue. + + When `sidekiq-cluster` is only running on a single node, make sure that at least + one process is running on all queues using `*`. This ensures a process + automatically picks up jobs in queues created in the future, + including queues that have dedicated processes. + + If `sidekiq-cluster` is running on more than one node, you can also use + [`--negate`](#negate-settings) and list all the queues that are already being + processed. + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +To view the Sidekiq processes in GitLab: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Monitoring > Background Jobs**. + +## Negate settings + +To have the Sidekiq process work on every queue **except** the ones +you list. In this example, we exclude all import-related jobs from a Sidekiq node: + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + sidekiq['negate'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=importers" + ] + ``` + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Queue selector + +> - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in GitLab 12.8. +> - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. +> - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. + +In addition to selecting queues by name, as above, the `queue_selector` option +allows queue groups to be selected in a more general way using a +[worker matching query](extra_sidekiq_routing.md#worker-matching-query). After `queue_selector` +is set, all `queue_groups` must follow the aforementioned syntax. + +In `/etc/gitlab/gitlab.rb`: + +```ruby +sidekiq['enable'] = true +sidekiq['queue_selector'] = true +sidekiq['queue_groups'] = [ + # Run all non-CPU-bound queues that are high urgency + 'resource_boundary!=cpu&urgency=high', + # Run all continuous integration and pages queues that are not high urgency + 'feature_category=continuous_integration,pages&urgency!=high', + # Run all queues + '*' +] +``` + +## Ignore all import queues + +When [importing from GitHub](../../user/project/import/github.md) or +other sources, Sidekiq might use all of its resources to perform those +operations. To set up two separate `sidekiq-cluster` processes, where +one only processes imports and the other processes all other queues: + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = true + sidekiq['queue_groups'] = [ + "feature_category=importers", + "feature_category!=importers" + ] + ``` + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Number of threads + +By default each process defined under `sidekiq` starts with a +number of threads that equals the number of queues, plus one spare thread. +For example, a process that handles the `process_commit` and `post_receive` +queues uses three threads in total. + +These thread run inside a single Ruby process, and each process +can only use a single CPU core. The usefulness of threading depends +on the work having some external dependencies to wait on, like database queries or +HTTP requests. Most Sidekiq deployments benefit from this threading, and when +running fewer queues in a process, increasing the thread count might be +even more desirable to make the most effective use of CPU resources. + +### Manage thread counts explicitly + +The correct maximum thread count (also called concurrency) depends on the workload. +Typical values range from `1` for highly CPU-bound tasks to `15` or higher for mixed +low-priority work. A reasonable starting range is `15` to `25` for a non-specialized +deployment. + +You can find example values used by GitLab.com by searching for `concurrency:` in +[the Helm charts](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/blob/master/releases/gitlab/values/gprd.yaml.gotmpl). +The values vary according to the work each specific deployment of Sidekiq does. +Any other specialized deployments with processes dedicated to specific queues should +have the concurrency tuned according to: +have the concurrency tuned according to: + +- The CPU usage of each type of process. +- The throughput achieved. + +Each thread requires a Redis connection, so adding threads may increase Redis +latency and potentially cause client timeouts. See the +[Sidekiq documentation about Redis](https://github.com/mperham/sidekiq/wiki/Using-Redis) for more +details. + +#### When running Sidekiq cluster (default) + +Running Sidekiq cluster is the default in GitLab 13.0 and later. + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + sidekiq['min_concurrency'] = 15 + sidekiq['max_concurrency'] = 25 + ``` + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +`min_concurrency` and `max_concurrency` are independent; one can be set without +the other. Setting `min_concurrency` to `0` disables the limit. + +For each queue group, let `N` be one more than the number of queues. The +concurrency is set to: + +1. `N`, if it's between `min_concurrency` and `max_concurrency`. +1. `max_concurrency`, if `N` exceeds this value. +1. `min_concurrency`, if `N` is less than this value. + +If `min_concurrency` is equal to `max_concurrency`, then this value is used +regardless of the number of queues. + +When `min_concurrency` is greater than `max_concurrency`, it is treated as +being equal to `max_concurrency`. + +#### When running a single Sidekiq process + +Running a single Sidekiq process is the default in GitLab 12.10 and earlier. + +WARNING: +Running Sidekiq directly was removed in GitLab +[14.0](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240). + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + sidekiq['cluster'] = false + sidekiq['concurrency'] = 25 + ``` + +1. Save the file and reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +This sets the concurrency (number of threads) for the Sidekiq process. + +## Modify the check interval + +To modify `sidekiq-cluster`'s health check interval for the additional Sidekiq processes: + +1. Edit `/etc/gitlab/gitlab.rb` and add (the value can be any integer number of seconds): + + ```ruby + sidekiq['interval'] = 5 + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + +## Troubleshoot using the CLI + +WARNING: +It's recommended to use `/etc/gitlab/gitlab.rb` to configure the Sidekiq processes. +If you experience a problem, you should contact GitLab support. Use the command +line at your own risk. + +For debugging purposes, you can start extra Sidekiq processes by using the command +`/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster`. This command +takes arguments using the following syntax: + +```shell +/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster [QUEUE,QUEUE,...] [QUEUE, ...] +``` + +Each separate argument denotes a group of queues that have to be processed by a +Sidekiq process. Multiple queues can be processed by the same process by +separating them with a comma instead of a space. + +Instead of a queue, a queue namespace can also be provided, to have the process +automatically listen on all queues in that namespace without needing to +explicitly list all the queue names. For more information about queue namespaces, +see the relevant section in the +[Sidekiq development documentation](../../development/sidekiq/index.md#queue-namespaces). + +For example, say you want to start 2 extra processes: one to process the +`process_commit` queue, and one to process the `post_receive` queue. This can be +done as follows: + +```shell +/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit post_receive +``` + +If you instead want to start one process processing both queues, you'd use the +following syntax: + +```shell +/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive +``` + +If you want to have one Sidekiq process dealing with the `process_commit` and +`post_receive` queues, and one process to process the `gitlab_shell` queue, +you'd use the following: + +```shell +/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster process_commit,post_receive gitlab_shell +``` + +### Monitor the `sidekiq-cluster` command + +The `sidekiq-cluster` command does not terminate once it has started the desired +amount of Sidekiq processes. Instead, the process continues running and +forwards any signals to the child processes. This allows you to stop all +Sidekiq processes as you send a signal to the `sidekiq-cluster` process, +instead of having to send it to the individual processes. + +If the `sidekiq-cluster` process crashes or receives a `SIGKILL`, the child +processes terminate themselves after a few seconds. This ensures you don't +end up with zombie Sidekiq processes. + +This allows you to monitor the processes by hooking up +`sidekiq-cluster` to your supervisor of choice (for example, runit). + +If a child process died the `sidekiq-cluster` command signals all remaining +process to terminate, then terminate itself. This removes the need for +`sidekiq-cluster` to re-implement complex process monitoring/restarting code. +Instead you should make sure your supervisor restarts the `sidekiq-cluster` +process whenever necessary. + +### PID files + +The `sidekiq-cluster` command can store its PID in a file. By default no PID +file is written, but this can be changed by passing the `--pidfile` option to +`sidekiq-cluster`. For example: + +```shell +/opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit +``` + +Keep in mind that the PID file contains the PID of the `sidekiq-cluster` +command and not the PIDs of the started Sidekiq processes. + +### Environment + +The Rails environment can be set by passing the `--environment` flag to the +`sidekiq-cluster` command, or by setting `RAILS_ENV` to a non-empty value. The +default value can be found in `/opt/gitlab/etc/gitlab-rails/env/RAILS_ENV`. diff --git a/doc/administration/sidekiq/extra_sidekiq_routing.md b/doc/administration/sidekiq/extra_sidekiq_routing.md new file mode 100644 index 00000000000..ebbf4059290 --- /dev/null +++ b/doc/administration/sidekiq/extra_sidekiq_routing.md @@ -0,0 +1,180 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Queue routing rules **(FREE SELF)** + +When the number of Sidekiq jobs increases to a certain scale, the system faces +some scalability issues. One of them is that the length of the queue tends to get +longer. High-urgency jobs have to wait longer until other less urgent jobs +finish. This head-of-line blocking situation may eventually affect the +responsiveness of the system, especially critical actions. In another scenario, +the performance of some jobs is degraded due to other long running or CPU-intensive jobs +(computing or rendering ones) in the same machine. + +To counter the aforementioned issues, one effective solution is to split +Sidekiq jobs into different queues and assign machines handling each queue +exclusively. For example, all CPU-intensive jobs could be routed to the +`cpu-bound` queue and handled by a fleet of CPU optimized instances. The queue +topology differs between companies depending on the workloads and usage +patterns. Therefore, GitLab supports a flexible mechanism for the +administrator to route the jobs based on their characteristics. + +As an alternative to [Queue selector](extra_sidekiq_processes.md#queue-selector), which +configures Sidekiq cluster to listen to a specific set of workers or queues, +GitLab also supports routing a job from a worker to the desired queue when it +is scheduled. Sidekiq clients try to match a job against a configured list of +routing rules. Rules are evaluated from first to last, and as soon as we find a +match for a given worker we stop processing for that worker (first match wins). +If the worker doesn't match any rule, it falls back to the queue name generated +from the worker name. + +By default, if the routing rules are not configured (or denoted with an empty +array), all the jobs are routed to the queue generated from the worker name. + +## Example configuration + +In `/etc/gitlab/gitlab.rb`: + +```ruby +sidekiq['routing_rules'] = [ + # Do not re-route workers that require their own queue + ['tags=needs_own_queue', nil], + # Route all non-CPU-bound workers that are high urgency to `high-urgency` queue + ['resource_boundary!=cpu&urgency=high', 'high-urgency'], + # Route all database, gitaly and global search workers that are throttled to `throttled` queue + ['feature_category=database,gitaly,global_search&urgency=throttled', 'throttled'], + # Route all workers having contact with outside work to a `network-intenstive` queue + ['has_external_dependencies=true|feature_category=hooks|tags=network', 'network-intensive'], + # Route all import workers to the queues generated by the worker name, for + # example, JiraImportWorker to `jira_import`, SVNWorker to `svn_worker` + ['feature_category=import', nil], + # Wildcard matching, route the rest to `default` queue + ['*', 'default'] +] +``` + +The routing rules list is an order-matter array of tuples of query and +corresponding queue: + +- The query is following a [worker matching query](#worker-matching-query) syntax. +- The `<queue_name>` must be a valid Sidekiq queue name. If the queue name + is `nil`, or an empty string, the worker is routed to the queue generated + by the name of the worker instead. + +The query supports wildcard matching `*`, which matches all workers. As a +result, the wildcard query must stay at the end of the list or the rules after it +are ignored. + +NOTE: +Mixing queue routing rules and queue selectors requires care to +ensure all jobs that are scheduled and picked up by appropriate Sidekiq +workers. + +## Worker matching query + +GitLab provides a query syntax to match a worker based on its +attributes. This query syntax is employed by both +[Queue routing rules](#queue-routing-rules) and +[Queue selector](extra_sidekiq_processes.md#queue-selector). A query includes two +components: + +- Attributes that can be selected. +- Operators used to construct a query. + +### Available attributes + +> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1 (`tags`). + +Queue matching query works upon the worker attributes, described in +[Sidekiq style guide](../../development/sidekiq/index.md). We support querying +based on a subset of worker attributes: + +- `feature_category` - the + [GitLab feature category](https://about.gitlab.com/direction/maturity/#category-maturity) the + queue belongs to. For example, the `merge` queue belongs to the + `source_code_management` category. +- `has_external_dependencies` - whether or not the queue connects to external + services. For example, all importers have this set to `true`. +- `urgency` - how important it is that this queue's jobs run + quickly. Can be `high`, `low`, or `throttled`. For example, the + `authorized_projects` queue is used to refresh user permissions, and + is `high` urgency. +- `worker_name` - the worker name. Use this attribute to select a specific worker. +- `name` - the queue name generated from the worker name. Use this attribute to select a specific queue. Because this is generated from + the worker name, it does not change based on the result of other routing + rules. +- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or + `unknown`. For example, the `ProjectExportWorker` is memory bound as it has + to load data in memory before saving it for export. +- `tags` - short-lived annotations for queues. These are expected to frequently + change from release to release, and may be removed entirely. + +`has_external_dependencies` is a boolean attribute: only the exact +string `true` is considered true, and everything else is considered +false. + +`tags` is a set, which means that `=` checks for intersecting sets, and +`!=` checks for disjoint sets. For example, `tags=a,b` selects queues +that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have +neither of those tags. + +The attributes of each worker are hard-coded in the source code. For +convenience, we generate a +[list of all available attributes in GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml) +and a +[list of all available attributes in GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/all_queues.yml). + +### Available operators + +`queue_selector` supports the following operators, listed from highest +to lowest precedence: + +- `|` - the logical `OR` operator. For example, `query_a|query_b` (where `query_a` + and `query_b` are queries made up of the other operators here) includes + queues that match either query. +- `&` - the logical `AND` operator. For example, `query_a&query_b` (where + `query_a` and `query_b` are queries made up of the other operators here) will + only include queues that match both queries. +- `!=` - the `NOT IN` operator. For example, `feature_category!=issue_tracking` + excludes all queues from the `issue_tracking` feature category. +- `=` - the `IN` operator. For example, `resource_boundary=cpu` includes all + queues that are CPU bound. +- `,` - the concatenate set operator. For example, + `feature_category=continuous_integration,pages` includes all queues from + either the `continuous_integration` category or the `pages` category. This + example is also possible using the OR operator, but allows greater brevity, as + well as being lower precedence. + +The operator precedence for this syntax is fixed: it's not possible to make `AND` +have higher precedence than `OR`. + +[In GitLab 12.9](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26594) and +later, as with the standard queue group syntax above, a single `*` as the +entire queue group selects all queues. + +### Migration + +After the Sidekiq routing rules are changed, administrators must take care +with the migration to avoid losing jobs entirely, especially in a system with +long queues of jobs. The migration can be done by following the migration steps +mentioned in [Sidekiq job migration](sidekiq_job_migration.md) + +### Workers that cannot be migrated + +Some workers cannot share a queue with other workers - typically because +they check the size of their own queue - and so must be excluded from +this process. We recommend excluding these from any further worker +routing by adding a rule to keep them in their own queue, for example: + +```ruby +sidekiq['routing_rules'] = [ + ['tags=needs_own_queue', nil], + # ... +] +``` + +These queues must also be included in at least one +[Sidekiq queue group](extra_sidekiq_processes.md#start-multiple-processes). diff --git a/doc/administration/troubleshooting/img/sidekiq_flamegraph.png b/doc/administration/sidekiq/img/sidekiq_flamegraph.png Binary files differindex 89d6e8da3ce..89d6e8da3ce 100644 --- a/doc/administration/troubleshooting/img/sidekiq_flamegraph.png +++ b/doc/administration/sidekiq/img/sidekiq_flamegraph.png diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md new file mode 100644 index 00000000000..f4e67fcb6dd --- /dev/null +++ b/doc/administration/sidekiq/index.md @@ -0,0 +1,403 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Configure an external Sidekiq instance **(FREE SELF)** + +You can configure an external Sidekiq instance by using the Sidekiq that's bundled in the GitLab package. Sidekiq requires connection to the Redis, +PostgreSQL, and Gitaly instances. + +## Configure TCP access for PostgreSQL, Gitaly, and Redis + +By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. To change this: + +1. Edit the `/etc/gitlab/gitlab.rb` file on your GitLab instance, and add the following: + + ```ruby + + ## PostgreSQL + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Add the Sidekiq nodes to PostgreSQL's trusted addresses. + # In the following example, 10.10.1.30/32 is the private IP + # of the Sidekiq server. + postgresql['md5_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32) + postgresql['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 10.10.1.30/32) + + ## Gitaly + + # Make Gitaly accept connections on all network interfaces + gitaly['listen_addr'] = "0.0.0.0:8075" + ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network + ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token + gitaly['auth_token'] = 'abc123secret' + praefect['auth_token'] = 'abc123secret' + gitlab_rails['gitaly_token'] = 'abc123secret' + + ## Redis configuration + + redis['bind'] = '0.0.0.0' + redis['port'] = 6379 + # Password to Authenticate Redis + redis['password'] = 'redis-password-goes-here' + gitlab_rails['redis_password'] = 'redis-password-goes-here' + + gitlab_rails['auto_migrate'] = false + ``` + +1. Run `reconfigure`: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Restart the `PostgreSQL` server: + + ```shell + sudo gitlab-ctl restart postgresql + ``` + +1. After the restart, set `auto_migrate` to `true` or comment to use the default settings: + + ```ruby + gitlab_rails['auto_migrate'] = true + ``` + +1. Run `reconfigure` again: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Set up Sidekiq instance + +1. SSH into the Sidekiq server. + +1. Confirm that you can access the PostgreSQL, Gitaly, and Redis ports: + + ```shell + telnet <GitLab host> 5432 # PostgreSQL + telnet <GitLab host> 8075 # Gitaly + telnet <GitLab host> 6379 # Redis + ``` + +1. [Download and install](https://about.gitlab.com/install/) the Omnibus GitLab package + using steps 1 and 2. **Do not complete any other steps.** + +1. Copy the `/etc/gitlab/gitlab.rb` file from the GitLab instance and add the following settings. Make sure + to replace them with your values: + +<!-- +Updates to example must be made at: +- https://gitlab.com/gitlab-org/gitlab/blob/master/doc/administration/sidekiq.md +- all reference architecture pages +--> + + ```ruby + ######################################## + ##### Services Disabled ### + ######################################## + # + # When running GitLab on just one server, you have a single `gitlab.rb` + # to enable all services you want to run. + # When running GitLab on N servers, you have N `gitlab.rb` files. + # Enable only the services you want to run on each + # specific server, while disabling all others. + # + gitaly['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + puma['enable'] = false + gitlab_workhorse['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + grafana['enable'] = false + gitlab_exporter['enable'] = false + gitlab_kas['enable'] = false + + ## + ## To maintain uniformity of links across nodes, the + ## `external_url` on the Sidekiq server should point to the external URL that users + ## use to access GitLab. This can be either: + ## + ## - The `external_url` set on your application server. + ## - The URL of a external load balancer, which routes traffic to the GitLab application server. + ## + external_url 'https://gitlab.example.com' + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + gitlab_rails['internal_api_url'] = 'GITLAB_URL' + gitlab_shell['secret_token'] = 'SHELL_TOKEN' + + ######################################## + #### Redis ### + ######################################## + + ## Must be the same in every sentinel node. + redis['master_name'] = 'gitlab-redis' # Required if you have setup redis cluster + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = '<redis_master_password>' + + ### If redis is running on the main Gitlab instance and you have opened the TCP port as above add the following + gitlab_rails['redis_host'] = '<gitlab_host>' + gitlab_rails['redis_port'] = 6379 + + ####################################### + ### Gitaly ### + ####################################### + + ## Replace <gitaly_token> with the one you set up, see + ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token + git_data_dirs({ + "default" => { + "gitaly_address" => "tcp://<gitlab_host>:8075", + "gitaly_token" => "<gitaly_token>" + } + }) + + ####################################### + ### Postgres ### + ####################################### + + # Replace <database_host> and <database_password> + gitlab_rails['db_host'] = '<database_host>' + gitlab_rails['db_port'] = '5432' + gitlab_rails['db_password'] = '<database_password>' + ## Prevent database migrations from running on upgrade automatically + gitlab_rails['auto_migrate'] = false + + ####################################### + ### Sidekiq configuration ### + ####################################### + sidekiq['enable'] = true + sidekiq['listen_address'] = "0.0.0.0" + + ## Set number of Sidekiq queue processes to the same number as available CPUs + sidekiq['queue_groups'] = ['*'] * 4 + + ## Set number of Sidekiq threads per queue process to the recommend number of 10 + sidekiq['max_concurrency'] = 10 + ``` + +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the GitLab instance and replace the file in the Sidekiq instance. + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. Restart the Sidekiq instance after completing the process and finishing the database migrations. + +## Configure multiple Sidekiq nodes with shared storage + +If you run multiple Sidekiq nodes with a shared file storage, such as NFS, you must +specify the UIDs and GIDs to ensure they match between servers. Specifying the UIDs +and GIDs prevents permissions issues in the file system. This advice is similar to the +[advice for Geo setups](../geo/replication/multiple_servers.md#step-4-configure-the-frontend-application-nodes-on-the-geo-secondary-site). + +To set up multiple Sidekiq nodes: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Configure the Container Registry when using an external Sidekiq + +If you're using the Container Registry and it's running on a different +node than Sidekiq, follow the steps below. + +1. Edit `/etc/gitlab/gitlab.rb`, and configure the registry URL: + + ```ruby + registry_external_url 'https://registry.example.com' + gitlab_rails['registry_api_url'] = "https://registry.example.com" + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. In the instance where Container Registry is hosted, copy the `registry.key` + file to the Sidekiq node. + +## Configure the Sidekiq metrics server + +If you want to collect Sidekiq metrics, enable the Sidekiq metrics server. +To make metrics available from `localhost:8082/metrics`: + +To configure the metrics server: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['metrics_enabled'] = true + sidekiq['listen_address'] = "localhost" + sidekiq['listen_port'] = "8082" + + # Optionally log all the metrics server logs to log/sidekiq_exporter.log + sidekiq['exporter_log_enabled'] = true + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +### Enable HTTPS + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364771) in GitLab 15.2. + +To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings: + +1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines: + + ```ruby + sidekiq['exporter_tls_enabled'] = true + sidekiq['exporter_tls_cert_path'] = "/path/to/certificate.pem" + sidekiq['exporter_tls_key_path'] = "/path/to/private-key.pem" + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +When TLS is enabled, the same `port` and `address` are used as described above. +The metrics server cannot serve both HTTP and HTTPS at the same time. + +## Configure health checks + +If you use health check probes to observe Sidekiq, enable the Sidekiq health check server. +To make health checks available from `localhost:8092`: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + sidekiq['health_checks_enabled'] = true + sidekiq['health_checks_listen_address'] = "localhost" + sidekiq['health_checks_listen_port'] = "8092" + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +For more information about health checks, see the [Sidekiq health check page](sidekiq_health_check.md). + +## Configure LDAP and user or group synchronization + +If you use LDAP for user and group management, you must add the LDAP configuration to your Sidekiq node as well as the LDAP +synchronization worker. If the LDAP configuration and LDAP synchronization worker are not applied to your Sidekiq node, +users and groups are not automatically synchronized. + +For more information about configuring LDAP for GitLab, see: + +- [GitLab LDAP configuration documentation](../auth/ldap/index.md#configure-ldap) +- [LDAP synchronization documentation](../auth/ldap/ldap_synchronization.md#adjust-ldap-user-sync-schedule) + +To enable LDAP with the synchronization worker for Sidekiq: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_enabled'] = true + gitlab_rails['prevent_ldap_sign_in'] = false + gitlab_rails['ldap_servers'] = { + 'main' => { + 'label' => 'LDAP', + 'host' => 'ldap.mydomain.com', + 'port' => 389, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'verify_certificates' => true, + 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', + 'password' => '_the_password_of_the_bind_user', + 'tls_options' => { + 'ca_file' => '', + 'ssl_version' => '', + 'ciphers' => '', + 'cert' => '', + 'key' => '' + }, + 'timeout' => 10, + 'active_directory' => true, + 'allow_username_or_email_login' => false, + 'block_auto_created_users' => false, + 'base' => 'dc=example,dc=com', + 'user_filter' => '', + 'attributes' => { + 'username' => ['uid', 'userid', 'sAMAccountName'], + 'email' => ['mail', 'email', 'userPrincipalName'], + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + }, + 'lowercase_usernames' => false, + + # Enterprise Edition only + # https://docs.gitlab.com/ee/administration/auth/ldap/ldap_synchronization.html + 'group_base' => '', + 'admin_group' => '', + 'external_groups' => [], + 'sync_ssh_keys' => false + } + } + gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## Disable Rugged + +Calls into Rugged, Ruby bindings for `libgit2`, [lock the Sidekiq processes's GVL](https://silverhammermba.github.io/emberb/c/#c-in-ruby-threads), +blocking all jobs on that worker from proceeding. If Rugged calls performed by Sidekiq are slow, this can cause significant delays in +background task processing. + +By default, Rugged is used when Git repository data is stored on local storage or on an NFS mount. +[Using Rugged is recommended when using NFS](../nfs.md#improving-nfs-performance-with-gitlab), but if +you are using local storage, disabling Rugged can improve Sidekiq performance: + +```shell +sudo gitlab-rake gitlab:features:disable_rugged +``` + +## Related topics + +- [Extra Sidekiq processes](extra_sidekiq_processes.md) +- [Extra Sidekiq routing](extra_sidekiq_routing.md) +- [Sidekiq health checks](sidekiq_health_check.md) +- [Using the GitLab-Sidekiq chart](https://docs.gitlab.com/charts/charts/gitlab/sidekiq/) + +## Troubleshooting + +See our [administrator guide to troubleshooting Sidekiq](sidekiq_troubleshooting.md). diff --git a/doc/administration/sidekiq/sidekiq_health_check.md b/doc/administration/sidekiq/sidekiq_health_check.md new file mode 100644 index 00000000000..3477320a2ac --- /dev/null +++ b/doc/administration/sidekiq/sidekiq_health_check.md @@ -0,0 +1,58 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Sidekiq Health Check **(FREE SELF)** + +GitLab provides liveness and readiness probes to indicate service health and +reachability to the Sidekiq cluster. These endpoints +[can be provided to schedulers like Kubernetes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +to hold traffic until the system is ready or restart the container as needed. + +The health check server can be set up when [configuring Sidekiq](index.md). + +## Readiness + +The readiness probe checks whether the Sidekiq workers are ready to process jobs. + +```plaintext +GET /readiness +``` + +If the server is bound to `localhost:8092`, the process cluster can be probed for readiness as follows: + +```shell +curl "http://localhost:8092/readiness" +``` + +On success, the endpoint returns a `200` HTTP status code, and a response like the following: + +```json +{ + "status": "ok" +} +``` + +## Liveness + +Checks whether the Sidekiq cluster is running. + +```plaintext +GET /liveness +``` + +If the server is bound to `localhost:8092`, the process cluster can be probed for liveness as follows: + +```shell +curl "http://localhost:8092/liveness" +``` + +On success, the endpoint returns a `200` HTTP status code, and a response like the following: + +```json +{ + "status": "ok" +} +``` diff --git a/doc/administration/sidekiq/sidekiq_job_migration.md b/doc/administration/sidekiq/sidekiq_job_migration.md new file mode 100644 index 00000000000..a3bc8b2959a --- /dev/null +++ b/doc/administration/sidekiq/sidekiq_job_migration.md @@ -0,0 +1,40 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Sidekiq job migration **(FREE SELF)** + +WARNING: +This operation should be very uncommon. We do not recommend it for the vast majority of GitLab instances. + +Sidekiq routing rules allow administrators to re-route certain background jobs from their regular queue to an alternative queue. By default, GitLab uses one queue per background job type. GitLab has over 400 background job types, and so correspondingly it has over 400 queues. + +Most administrators do not need to change this setting. In some cases with particularly large background job processing workloads, Redis performance may suffer due to the number of queues that GitLab listens to. + +If the Sidekiq routing rules are changed, administrators need to take care with the migration to avoid losing jobs entirely. The basic migration steps are: + +1. Listen to both the old and new queues. +1. Update the routing rules. +1. Wait until there are no publishers dispatching jobs to the old queues. +1. Run the [Rake tasks for future jobs](#future-jobs). +1. Wait for the old queues to be empty. +1. Stop listening to the old queues. + +## Future jobs + +Step 4 involves rewriting some Sidekiq job data for jobs that are already stored in Redis, but due to run in future. There are two sets of jobs to run in future: scheduled jobs and jobs to be retried. We provide a separate Rake task to migrate each set: + +- `gitlab:sidekiq:migrate_jobs:retry` for jobs to be retried. +- `gitlab:sidekiq:migrate_jobs:scheduled` for scheduled jobs. + +Most of the time, running both at the same time is the correct choice. There are two separate tasks to allow for more fine-grained control where needed. To run both at once: + +```shell +# omnibus-gitlab +sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule + +# source installations +bundle exec rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule RAILS_ENV=production +``` diff --git a/doc/administration/sidekiq/sidekiq_memory_killer.md b/doc/administration/sidekiq/sidekiq_memory_killer.md new file mode 100644 index 00000000000..12381808523 --- /dev/null +++ b/doc/administration/sidekiq/sidekiq_memory_killer.md @@ -0,0 +1,82 @@ +--- +stage: Data Stores +group: Memory +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Sidekiq MemoryKiller **(FREE SELF)** + +The GitLab Rails application code suffers from memory leaks. For web requests +this problem is made manageable using +[`puma-worker-killer`](https://github.com/schneems/puma_worker_killer) which +restarts Puma worker processes if it exceeds a memory limit. The Sidekiq +MemoryKiller applies the same approach to the Sidekiq processes used by GitLab +to process background jobs. + +Unlike puma-worker-killer, which is enabled by default for all GitLab +installations of GitLab 13.0 and later, the Sidekiq MemoryKiller is enabled by default +_only_ for Omnibus packages. The reason for this is that the MemoryKiller +relies on runit to restart Sidekiq after a memory-induced shutdown and GitLab +installations from source do not all use runit or an equivalent. + +With the default settings, the MemoryKiller causes a Sidekiq restart no +more often than once every 15 minutes, with the restart causing about one +minute of delay for incoming background jobs. + +Some background jobs rely on long-running external processes. To ensure these +are cleanly terminated when Sidekiq is restarted, each Sidekiq process should be +run as a process group leader (for example, using `chpst -P`). If using Omnibus or the +`bin/background_jobs` script with `runit` installed, this is handled for you. + +## Configuring the MemoryKiller + +The MemoryKiller is controlled using environment variables. + +- `SIDEKIQ_DAEMON_MEMORY_KILLER`: defaults to 1. When set to 0, the MemoryKiller + works in _legacy_ mode. Otherwise, the MemoryKiller works in _daemon_ mode. + + In _legacy_ mode, the MemoryKiller checks the Sidekiq process RSS + ([Resident Set Size](https://github.com/mperham/sidekiq/wiki/Memory#rss)) + after each job. + + In _daemon_ mode, the MemoryKiller checks the Sidekiq process RSS every 3 seconds + (defined by `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL`). + +- `SIDEKIQ_MEMORY_KILLER_MAX_RSS` (KB): if this variable is set, and its value is greater + than 0, the MemoryKiller is enabled. Otherwise the MemoryKiller is disabled. + + `SIDEKIQ_MEMORY_KILLER_MAX_RSS` defines the Sidekiq process allowed RSS. + + In _legacy_ mode, if the Sidekiq process exceeds the allowed RSS then an irreversible + delayed graceful restart is triggered. The restart of Sidekiq happens + after `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` seconds. + + In _daemon_ mode, if the Sidekiq process exceeds the allowed RSS for longer than + `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` the graceful restart is triggered. If the + Sidekiq process go below the allowed RSS within `SIDEKIQ_MEMORY_KILLER_GRACE_TIME`, + the restart is aborted. + + The default value for Omnibus packages is set + [in the Omnibus GitLab repository](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb). + +- `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS` (KB): is used by _daemon_ mode. If the Sidekiq + process RSS (expressed in kilobytes) exceeds `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS`, + an immediate graceful restart of Sidekiq is triggered. + +- `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL`: used in _daemon_ mode to define how + often to check process RSS, default to 3 seconds. + +- `SIDEKIQ_MEMORY_KILLER_GRACE_TIME`: defaults to 900 seconds (15 minutes). + The usage of this variable is described as part of `SIDEKIQ_MEMORY_KILLER_MAX_RSS`. + +- `SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT`: defaults to 30 seconds. This defines the + maximum time allowed for all Sidekiq jobs to finish. No new jobs are accepted + during that time, and the process exits as soon as all jobs finish. + + If jobs do not finish during that time, the MemoryKiller interrupts all currently + running jobs by sending `SIGTERM` to the Sidekiq process. + + If the process hard shutdown/restart is not performed by Sidekiq, + the Sidekiq process is forcefully terminated after + `Sidekiq.options[:timeout] + 2` seconds. An external supervision mechanism + (for example, runit) must restart Sidekiq afterwards. diff --git a/doc/administration/sidekiq/sidekiq_troubleshooting.md b/doc/administration/sidekiq/sidekiq_troubleshooting.md new file mode 100644 index 00000000000..9bc8e473409 --- /dev/null +++ b/doc/administration/sidekiq/sidekiq_troubleshooting.md @@ -0,0 +1,381 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Troubleshooting Sidekiq **(FREE SELF)** + +Sidekiq is the background job processor GitLab uses to asynchronously run +tasks. When things go wrong it can be difficult to troubleshoot. These +situations also tend to be high-pressure because a production system job queue +may be filling up. Users will notice when this happens because new branches +may not show up and merge requests may not be updated. The following are some +troubleshooting steps to help you diagnose the bottleneck. + +GitLab administrators/users should consider working through these +debug steps with GitLab Support so the backtraces can be analyzed by our team. +It may reveal a bug or necessary improvement in GitLab. + +In any of the backtraces, be wary of suspecting cases where every +thread appears to be waiting in the database, Redis, or waiting to acquire +a mutex. This **may** mean there's contention in the database, for example, +but look for one thread that is different than the rest. This other thread +may be using all available CPU, or have a Ruby Global Interpreter Lock, +preventing other threads from continuing. + +## Log arguments to Sidekiq jobs + +[In GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44853) +some arguments passed to Sidekiq jobs are logged by default. +To avoid logging sensitive information (for instance, password reset tokens), +GitLab logs numeric arguments for all workers, with overrides for some specific +workers where their arguments are not sensitive. + +Example log output: + +```json +{"severity":"INFO","time":"2020-06-08T14:37:37.892Z","class":"AdminEmailsWorker","args":["[FILTERED]","[FILTERED]","[FILTERED]"],"retry":3,"queue":"admin_emails","backtrace":true,"jid":"9e35e2674ac7b12d123e13cc","created_at":"2020-06-08T14:37:37.373Z","meta.user":"root","meta.caller_id":"Admin::EmailsController#create","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:6dc94409cfdd4d77:9fbe19bdee865293:1","enqueued_at":"2020-06-08T14:37:37.410Z","pid":65011,"message":"AdminEmailsWorker JID-9e35e2674ac7b12d123e13cc: done: 0.48085 sec","job_status":"done","scheduling_latency_s":0.001012,"redis_calls":9,"redis_duration_s":0.004608,"redis_read_bytes":696,"redis_write_bytes":6141,"duration_s":0.48085,"cpu_s":0.308849,"completed_at":"2020-06-08T14:37:37.892Z","db_duration_s":0.010742} +{"severity":"INFO","time":"2020-06-08T14:37:37.894Z","class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::MailDeliveryJob","queue":"mailers","args":["[FILTERED]"],"retry":3,"backtrace":true,"jid":"e47a4f6793d475378432e3c8","created_at":"2020-06-08T14:37:37.884Z","meta.user":"root","meta.caller_id":"AdminEmailsWorker","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:29344de0f966446d:5c3b0e0e1bef987b:1","enqueued_at":"2020-06-08T14:37:37.885Z","pid":65011,"message":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper JID-e47a4f6793d475378432e3c8: start","job_status":"start","scheduling_latency_s":0.009473} +{"severity":"INFO","time":"2020-06-08T14:39:50.648Z","class":"NewIssueWorker","args":["455","1"],"retry":3,"queue":"new_issue","backtrace":true,"jid":"a24af71f96fd129ec47f5d1e","created_at":"2020-06-08T14:39:50.643Z","meta.user":"root","meta.project":"h5bp/html5-boilerplate","meta.root_namespace":"h5bp","meta.caller_id":"Projects::IssuesController#create","correlation_id":"f9UCZHqhuP7","uber-trace-id":"28f65730f99f55a3:a5d2b62dec38dffc:48ddd092707fa1b7:1","enqueued_at":"2020-06-08T14:39:50.646Z","pid":65011,"message":"NewIssueWorker JID-a24af71f96fd129ec47f5d1e: start","job_status":"start","scheduling_latency_s":0.001144} +``` + +When using [Sidekiq JSON logging](../logs/index.md#sidekiqlog), +arguments logs are limited to a maximum size of 10 kilobytes of text; +any arguments after this limit are discarded and replaced with a +single argument containing the string `"..."`. + +You can set `SIDEKIQ_LOG_ARGUMENTS` [environment variable](https://docs.gitlab.com/omnibus/settings/environment-variables.html) +to `0` (false) to disable argument logging. + +Example: + +```ruby +gitlab_rails['env'] = {"SIDEKIQ_LOG_ARGUMENTS" => "0"} +``` + +In GitLab 13.5 and earlier, set `SIDEKIQ_LOG_ARGUMENTS` to `1` to start logging arguments passed to Sidekiq. + +## Thread dump + +Send the Sidekiq process ID the `TTIN` signal to output thread +backtraces in the log file. + +```shell +kill -TTIN <sidekiq_pid> +``` + +Check in `/var/log/gitlab/sidekiq/current` or `$GITLAB_HOME/log/sidekiq.log` for +the backtrace output. The backtraces are lengthy and generally start with +several `WARN` level messages. Here's an example of a single thread's backtrace: + +```plaintext +2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: ActiveRecord::RecordNotFound: Couldn't find Note with 'id'=3375386 +2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/activerecord-4.2.5.2/lib/active_record/core.rb:155:in `find' +/opt/gitlab/embedded/service/gitlab-rails/app/workers/new_note_worker.rb:7:in `perform' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:150:in `execute_job' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:132:in `block (2 levels) in process' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:127:in `block in invoke' +/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/memory_killer.rb:17:in `call' +/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:129:in `block in invoke' +/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/arguments_logger.rb:6:in `call' +... +``` + +In some cases Sidekiq may be hung and unable to respond to the `TTIN` signal. +Move on to other troubleshooting methods if this happens. + +## Ruby profiling with `rbspy` + +[rbspy](https://rbspy.github.io) is an easy to use and low-overhead Ruby profiler that can be used to create +flamegraph-style diagrams of CPU usage by Ruby processes. + +No changes to GitLab are required to use it and it has no dependencies. To install it: + +1. Download the binary from the [`rbspy` releases page](https://github.com/rbspy/rbspy/releases). +1. Make the binary executable. + +To profile a Sidekiq worker for one minute, run: + +```shell +sudo ./rbspy record --pid <sidekiq_pid> --duration 60 --file /tmp/sidekiq_profile.svg +``` + + + +In this example of a flamegraph generated by `rbspy`, almost all of the Sidekiq process's time is spent in `rev_parse`, a native C +function in Rugged. In the stack, we can see `rev_parse` is being called by the `ExpirePipelineCacheWorker`. + +## Process profiling with `perf` + +Linux has a process profiling tool called `perf` that is helpful when a certain +process is eating up a lot of CPU. If you see high CPU usage and Sidekiq isn't +responding to the `TTIN` signal, this is a good next step. + +If `perf` is not installed on your system, install it with `apt-get` or `yum`: + +```shell +# Debian +sudo apt-get install linux-tools + +# Ubuntu (may require these additional Kernel packages) +sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` + +# Red Hat/CentOS +sudo yum install perf +``` + +Run `perf` against the Sidekiq PID: + +```shell +sudo perf record -p <sidekiq_pid> +``` + +Let this run for 30-60 seconds and then press Ctrl-C. Then view the `perf` report: + +```shell +$ sudo perf report + +# Sample output +Samples: 348K of event 'cycles', Event count (approx.): 280908431073 + 97.69% ruby nokogiri.so [.] xmlXPathNodeSetMergeAndClear + 0.18% ruby libruby.so.2.1.0 [.] objspace_malloc_increase + 0.12% ruby libc-2.12.so [.] _int_malloc + 0.10% ruby libc-2.12.so [.] _int_free +``` + +Above you see sample output from a `perf` report. It shows that 97% of the CPU is +being spent inside Nokogiri and `xmlXPathNodeSetMergeAndClear`. For something +this obvious you should then go investigate what job in GitLab would use +Nokogiri and XPath. Combine with `TTIN` or `gdb` output to show the +corresponding Ruby code where this is happening. + +## The GNU Project Debugger (`gdb`) + +`gdb` can be another effective tool for debugging Sidekiq. It gives you a little +more interactive way to look at each thread and see what's causing problems. + +Attaching to a process with `gdb` suspends the normal operation +of the process (Sidekiq does not process jobs while `gdb` is attached). + +Start by attaching to the Sidekiq PID: + +```shell +gdb -p <sidekiq_pid> +``` + +Then gather information on all the threads: + +```plaintext +info threads + +# Example output +30 Thread 0x7fe5fbd63700 (LWP 26060) 0x0000003f7cadf113 in poll () from /lib64/libc.so.6 +29 Thread 0x7fe5f2b3b700 (LWP 26533) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +28 Thread 0x7fe5f2a3a700 (LWP 26534) 0x0000003f7ce0ba5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +27 Thread 0x7fe5f2939700 (LWP 26535) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +26 Thread 0x7fe5f2838700 (LWP 26537) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +25 Thread 0x7fe5f2737700 (LWP 26538) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +24 Thread 0x7fe5f2535700 (LWP 26540) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +23 Thread 0x7fe5f2434700 (LWP 26541) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +22 Thread 0x7fe5f2232700 (LWP 26543) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 +21 Thread 0x7fe5f2131700 (LWP 26544) 0x00007fe5f7b570f0 in xmlXPathNodeSetMergeAndClear () +from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +... +``` + +If you see a suspicious thread, like the Nokogiri one above, you may want +to get more information: + +```plaintext +thread 21 +bt + +# Example output +#0 0x00007ff0d6afe111 in xmlXPathNodeSetMergeAndClear () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#1 0x00007ff0d6b0b836 in xmlXPathNodeCollectAndTest () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#2 0x00007ff0d6b09037 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#3 0x00007ff0d6b09017 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#4 0x00007ff0d6b092e0 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#5 0x00007ff0d6b0bc37 in xmlXPathRunEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#6 0x00007ff0d6b0be5f in xmlXPathEvalExpression () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so +#7 0x00007ff0d6a97dc3 in evaluate (argc=2, argv=0x1022d058, self=<value optimized out>) at xml_xpath_context.c:221 +#8 0x00007ff0daeab0ea in vm_call_cfunc_with_frame (th=0x1022a4f0, reg_cfp=0x1032b810, ci=<value optimized out>) at vm_insnhelper.c:1510 +``` + +To output a backtrace from all threads at once: + +```plaintext +set pagination off +thread apply all bt +``` + +Once you're done debugging with `gdb`, be sure to detach from the process and +exit: + +```plaintext +detach +exit +``` + +## Sidekiq kill signals + +TTIN was described above as the signal to print backtraces for logging, however +Sidekiq responds to other signals as well. For example, TSTP and TERM can be used +to gracefully shut Sidekiq down, see +[the Sidekiq Signals docs](https://github.com/mperham/sidekiq/wiki/Signals#ttin). + +## Check for blocking queries + +Sometimes the speed at which Sidekiq processes jobs can be so fast that it can +cause database contention. Check for blocking queries when backtraces above +show that many threads are stuck in the database adapter. + +The PostgreSQL wiki has details on the query you can run to see blocking +queries. The query is different based on PostgreSQL version. See +[Lock Monitoring](https://wiki.postgresql.org/wiki/Lock_Monitoring) for +the query details. + +## Managing Sidekiq queues + +It is possible to use [Sidekiq API](https://github.com/mperham/sidekiq/wiki/API) +to perform a number of troubleshooting steps on Sidekiq. + +These are the administrative commands and it should only be used if currently +administration interface is not suitable due to scale of installation. + +All these commands should be run using `gitlab-rails console`. + +### View the queue size + +```ruby +Sidekiq::Queue.new("pipeline_processing:build_queue").size +``` + +### Enumerate all enqueued jobs + +```ruby +queue = Sidekiq::Queue.new("chaos:chaos_sleep") +queue.each do |job| + # job.klass # => 'MyWorker' + # job.args # => [1, 2, 3] + # job.jid # => jid + # job.queue # => chaos:chaos_sleep + # job["retry"] # => 3 + # job.item # => { + # "class"=>"Chaos::SleepWorker", + # "args"=>[1000], + # "retry"=>3, + # "queue"=>"chaos:chaos_sleep", + # "backtrace"=>true, + # "queue_namespace"=>"chaos", + # "jid"=>"39bc482b823cceaf07213523", + # "created_at"=>1566317076.266069, + # "correlation_id"=>"c323b832-a857-4858-b695-672de6f0e1af", + # "enqueued_at"=>1566317076.26761}, + # } + + # job.delete if job.jid == 'abcdef1234567890' +end +``` + +### Enumerate currently running jobs + +```ruby +workers = Sidekiq::Workers.new +workers.each do |process_id, thread_id, work| + # process_id is a unique identifier per Sidekiq process + # thread_id is a unique identifier per thread + # work is a Hash which looks like: + # {"queue"=>"chaos:chaos_sleep", + # "payload"=> + # { "class"=>"Chaos::SleepWorker", + # "args"=>[1000], + # "retry"=>3, + # "queue"=>"chaos:chaos_sleep", + # "backtrace"=>true, + # "queue_namespace"=>"chaos", + # "jid"=>"b2a31e3eac7b1a99ff235869", + # "created_at"=>1566316974.9215662, + # "correlation_id"=>"e484fb26-7576-45f9-bf21-b99389e1c53c", + # "enqueued_at"=>1566316974.9229589}, + # "run_at"=>1566316974}], +end +``` + +### Remove Sidekiq jobs for given parameters (destructive) + +The general method to kill jobs conditionally is the following command, which +removes jobs that are queued but not started. Running jobs can not be killed. + +```ruby +queue = Sidekiq::Queue.new('<queue name>') +queue.each { |job| job.delete if <condition>} +``` + +Have a look at the section below for cancelling running jobs. + +In the method above, `<queue-name>` is the name of the queue that contains the jobs you want to delete and `<condition>` decides which jobs get deleted. + +Commonly, `<condition>` references the job arguments, which depend on the type of job in question. To find the arguments for a specific queue, you can have a look at the `perform` function of the related worker file, commonly found at `/app/workers/<queue-name>_worker.rb`. + +For example, `repository_import` has `project_id` as the job argument, while `update_merge_requests` has `project_id, user_id, oldrev, newrev, ref`. + +Arguments need to be referenced by their sequence ID using `job.args[<id>]` because `job.args` is a list of all arguments provided to the Sidekiq job. + +Here are some examples: + +```ruby +queue = Sidekiq::Queue.new('update_merge_requests') +# In this example, we want to remove any update_merge_requests jobs +# for the Project with ID 125 and ref `ref/heads/my_branch` +queue.each { |job| job.delete if job.args[0] == 125 and job.args[4] == 'ref/heads/my_branch' } +``` + +```ruby +# Cancelling jobs like: `RepositoryImportWorker.new.perform_async(100)` +id_list = [100] + +queue = Sidekiq::Queue.new('repository_import') +queue.each do |job| + job.delete if id_list.include?(job.args[0]) +end +``` + +### Remove specific job ID (destructive) + +```ruby +queue = Sidekiq::Queue.new('repository_import') +queue.each do |job| + job.delete if job.jid == 'my-job-id' +end +``` + +## Canceling running jobs (destructive) + +> Introduced in GitLab 12.3. + +This is highly risky operation and use it as last resort. +Doing that might result in data corruption, as the job +is interrupted mid-execution and it is not guaranteed +that proper rollback of transactions is implemented. + +```ruby +Gitlab::SidekiqDaemon::Monitor.cancel_job('job-id') +``` + +> This requires the Sidekiq to be run with `SIDEKIQ_MONITOR_WORKER=1` +> environment variable. + +To perform of the interrupt we use `Thread.raise` which +has number of drawbacks, as mentioned in [Why Ruby's Timeout is dangerous (and Thread.raise is terrifying)](https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/): + +> This is where the implications get interesting, and terrifying. This means that an exception can get raised: +> +> - during a network request (ok, as long as the surrounding code is prepared to catch Timeout::Error) +> - during the cleanup for the network request +> - during a rescue block +> - while creating an object to save to the database afterwards +> - in any of your code, regardless of whether it could have possibly raised an exception before +> +> Nobody writes code to defend against an exception being raised on literally any line. That's not even possible. So Thread.raise is basically like a sneak attack on your code that could result in almost anything. It would probably be okay if it were pure-functional code that did not modify any state. But this is Ruby, so that's unlikely :) diff --git a/doc/administration/sidekiq_health_check.md b/doc/administration/sidekiq_health_check.md index aaab2b7e086..3294eb663f2 100644 --- a/doc/administration/sidekiq_health_check.md +++ b/doc/administration/sidekiq_health_check.md @@ -1,58 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'sidekiq/sidekiq_health_check.md' +remove_date: '2022-11-11' --- -# Sidekiq Health Check **(FREE SELF)** +This document was moved to [another location](sidekiq/sidekiq_health_check.md). -GitLab provides liveness and readiness probes to indicate service health and -reachability to the Sidekiq cluster. These endpoints -[can be provided to schedulers like Kubernetes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) -to hold traffic until the system is ready or restart the container as needed. - -The health check server can be set up when [configuring Sidekiq](sidekiq.md). - -## Readiness - -The readiness probe checks whether the Sidekiq workers are ready to process jobs. - -```plaintext -GET /readiness -``` - -If the server is bound to `localhost:8092`, the process cluster can be probed for readiness as follows: - -```shell -curl "http://localhost:8092/readiness" -``` - -On success, the endpoint returns a `200` HTTP status code, and a response like the following: - -```json -{ - "status": "ok" -} -``` - -## Liveness - -Checks whether the Sidekiq cluster is running. - -```plaintext -GET /liveness -``` - -If the server is bound to `localhost:8092`, the process cluster can be probed for liveness as follows: - -```shell -curl "http://localhost:8092/liveness" -``` - -On success, the endpoint returns a `200` HTTP status code, and a response like the following: - -```json -{ - "status": "ok" -} -``` +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index 14649b82ba8..7a8d7774948 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -78,9 +78,9 @@ Terraform state files are stored locally, follow the steps below. ## Using object storage **(FREE SELF)** -Instead of storing Terraform state files on disk, we recommend the use of [one of the supported object -storage options](object_storage.md#options). This configuration relies on valid credentials to -be configured already. +Instead of storing Terraform state files on disk, we recommend the use of +[one of the supported object storage options](object_storage.md#options). +This configuration relies on valid credentials to be configured already. [Read more about using object storage with GitLab](object_storage.md). diff --git a/doc/administration/troubleshooting/elasticsearch.md b/doc/administration/troubleshooting/elasticsearch.md index 7ce09252680..7390f4bc816 100644 --- a/doc/administration/troubleshooting/elasticsearch.md +++ b/doc/administration/troubleshooting/elasticsearch.md @@ -1,404 +1,11 @@ --- -stage: Data Stores -group: Global Search -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../integration/advanced_search/elasticsearch_troubleshooting.md' +remove_date: '2022-11-02' --- -# Troubleshooting Elasticsearch **(PREMIUM SELF)** +This document was moved to [another location](../../integration/advanced_search/elasticsearch_troubleshooting.md). -To install and configure Elasticsearch, -visit the [administrator documentation](../../integration/advanced_search/elasticsearch.md). - -For troubleshooting, visit the -[administrator troubleshooting documentation](../../integration/advanced_search/elasticsearch_troubleshooting.md). - -Troubleshooting Elasticsearch requires: - -- Knowledge of common terms. -- Establishing within which category the problem fits. - -## Common terminology - -- **Lucene**: A full-text search library written in Java. -- **Near real time (NRT)**: Refers to the slight latency from the time to index a - document to the time when it becomes searchable. -- **Cluster**: A collection of one or more nodes that work together to hold all - the data, providing indexing and search capabilities. -- **Node**: A single server that works as part of a cluster. -- **Index**: A collection of documents that have somewhat similar characteristics. -- **Document**: A basic unit of information that can be indexed. -- **Shards**: Fully-functional and independent subdivisions of indices. Each shard is actually - a Lucene index. -- **Replicas**: Failover mechanisms that duplicate indices. - -## Troubleshooting workflows - -The type of problem will determine what steps to take. The possible troubleshooting workflows are for: - -- Search results. -- Indexing. -- Integration. -- Performance. -- Advanced Search Migrations. - -### Search Results workflow - -The following workflow is for Elasticsearch search results issues: - -```mermaid -graph TD; - B --> |No| B1 - B --> |Yes| B4 - B1 --> B2 - B2 --> B3 - B4 --> B5 - B5 --> |Yes| B6 - B5 --> |No| B7 - B7 --> B8 - B{Is GitLab using<br>Elasticsearch for<br>searching?} - B1[From the Admin Area, select<br>Integrations from the left<br>sidebar to ensure the settings<br>are correct.] - B2[Perform a search via<br>the rails console] - B3[If all settings are correct<br>and it still doesn't show Elasticsearch<br>doing the searches, escalate<br>to GitLab support.] - B4[Perform<br>the same search via the<br>Elasticsearch API] - B5{Are the results<br>the same?} - B6[This means it is working as intended.<br>Speak with GitLab support<br>to confirm if the issue lies with<br>the filters.] - B7[Check the index status of the project<br>containing the missing search<br>results.] - B8(Indexing Troubleshooting) -``` - -### Indexing workflow - -The following workflow is for Elasticsearch indexing issues: - -```mermaid -graph TD; - C --> |Yes| C1 - C1 --> |Yes| C2 - C1 --> |No| C3 - C3 --> |Yes| C4 - C3 --> |No| C5 - C --> |No| C6 - C6 --> |No| C10 - C7 --> |GitLab| C8 - C7 --> |Elasticsearch| C9 - C6 --> |Yes| C7 - C10 --> |No| C12 - C10 --> |Yes| C11 - C12 --> |Yes| C13 - C12 --> |No| C14 - C14 --> |Yes| C15 - C14 --> |No| C16 - C{Is the problem with<br>creating an empty<br>index?} - C1{Does the gitlab-production<br>index exist on the<br>Elasticsearch instance?} - C2(Try to manually<br>delete the index on the<br>Elasticsearch instance and<br>retry creating an empty index.) - C3{Can indices be made<br>manually on the Elasticsearch<br>instance?} - C4(Retry the creation of an empty index) - C5(It is best to speak with an<br>Elasticsearch admin concerning the<br>instance's inability to create indices.) - C6{Is the indexer presenting<br>errors during indexing?} - C7{Is the error a GitLab<br>error or an Elasticsearch<br>error?} - C8[Escalate to<br>GitLab support] - C9[You will want<br>to speak with an<br>Elasticsearch admin.] - C10{Does the index status<br>show 100%?} - C11[Escalate to<br>GitLab support] - C12{Does re-indexing the project<br> present any GitLab errors?} - C13[Rectify the GitLab errors and<br>restart troubleshooting, or<br>escalate to GitLab support.] - C14{Does re-indexing the project<br>present errors on the <br>Elasticsearch instance?} - C15[It would be best<br>to speak with an<br>Elasticsearch admin.] - C16[This is likely a bug/issue<br>in GitLab and will require<br>deeper investigation. Escalate<br>to GitLab support.] -``` - -### Integration workflow - -The following workflow is for Elasticsearch integration issues: - -```mermaid -graph TD; - D --> |No| D1 - D --> |Yes| D2 - D2 --> |No| D3 - D2 --> |Yes| D4 - D4 --> |No| D5 - D4 --> |Yes| D6 - D{Is the error concerning<br>the Go indexer?} - D1[It would be best<br>to speak with an<br>Elasticsearch admin.] - D2{Is the ICU development<br>package installed?} - D3>This package is required.<br>Install the package<br>and retry.] - D4{Is the error stemming<br>from the indexer?} - D5[This would indicate an OS level<br> issue. It would be best to<br>contact your sysadmin.] - D6[This is likely a bug/issue<br>in GitLab and will require<br>deeper investigation. Escalate<br>to GitLab support.] -``` - -### Performance workflow - -The following workflow is for Elasticsearch performance issues: - -```mermaid -graph TD; - F --> |Yes| F1 - F --> |No| F2 - F2 --> |No| F3 - F2 --> |Yes| F4 - F4 --> F5 - F5 --> |No| F6 - F5 --> |Yes| F7 - F{Is the Elasticsearch instance<br>running on the same server<br>as the GitLab instance?} - F1(This is not advised and will cause issues.<br>We recommend moving the Elasticsearch<br>instance to a different server.) - F2{Does the Elasticsearch<br>server have at least 8<br>GB of RAM and 2 CPU<br>cores?} - F3(According to Elasticsearch, a non-prod<br>server needs these as a base requirement.<br>Production often requires more. We recommend<br>you increase the server specifications.) - F4(Obtain the <br>cluster health information) - F5(Does it show the<br>status as green?) - F6(We recommend you speak with<br>an Elasticsearch admin<br>about implementing sharding.) - F7(Escalate to<br>GitLab support.) -``` - -### Advanced Search Migrations workflow - -```mermaid -graph TD; - D --> |No| D1 - D --> |Yes| D2 - D2 --> |No| D3 - D2 --> |Yes| D4 - D4 --> |No| D5 - D4 --> |Yes| D6 - D6 --> |No| D8 - D6 --> |Yes| D7 - - D{Is there a halted migration?} - D1[Migrations run in the<br>background and will<br>stop when completed.] - D2{Does the elasticsearch.log<br>file contain errors?} - D3[This is likely a bug/issue<br>in GitLab and will require<br>deeper investigation. Escalate<br>to GitLab support.] - D4{Have the errors<br>been addressed?} - D5[Have an Elasticsearch admin<br>review and address<br>the errors.] - D6{Has the migration<br>been retried?} - D7[This is likely a bug/issue<br>in GitLab and will require<br>deeper investigation. Escalate<br>to GitLab support.] - D8[Retry the migration from<br>the Admin > Settings ><br>Advanced Search UI.] -``` - -## Troubleshooting walkthrough - -Most Elasticsearch troubleshooting can be broken down into 4 categories: - -- [Troubleshooting search results](#troubleshooting-search-results) -- [Troubleshooting indexing](#troubleshooting-indexing) -- [Troubleshooting integration](#troubleshooting-integration) -- [Troubleshooting performance](#troubleshooting-performance) -- [Troubleshooting Advanced Search migrations](#troubleshooting-advanced-search-migrations) - -Generally speaking, if it does not fall into those four categories, it is either: - -- Something GitLab support needs to look into. -- Not a true Elasticsearch issue. - -Exercise caution. Issues that appear to be Elasticsearch problems can be OS-level issues. - -### Troubleshooting search results - -Troubleshooting search result issues is rather straight forward on Elasticsearch. - -The first step is to confirm GitLab is using Elasticsearch for the search function. -To do this: - -1. On the top bar, select **Menu > Admin**. -1. On the left sidebar, select **Settings > General**, and then confirm the - integration is enabled. -1. Confirm searches use Elasticsearch by accessing the rails console - (`sudo gitlab-rails console`) and running the following commands: - - ```rails - u = User.find_by_email('email_of_user_doing_search') - s = SearchService.new(u, {:search => 'search_term'}) - pp s.search_objects.class - ``` - -The output from the last command is the key here. If it shows: - -- `ActiveRecord::Relation`, **it is not** using Elasticsearch. -- `Kaminari::PaginatableArray`, **it is** using Elasticsearch. - -| Not using Elasticsearch | Using Elasticsearch | -|--------------------------|------------------------------| -| `ActiveRecord::Relation` | `Kaminari::PaginatableArray` | - -If all the settings look correct and it is still not using Elasticsearch for the search function, it is best to escalate to GitLab support. This could be a bug/issue. - -Moving past that, it is best to attempt the same [search via the Rails console](../../integration/advanced_search/elasticsearch_troubleshooting.md#i-indexed-all-the-repositories-but-i-cant-get-any-hits-for-my-search-term-in-the-ui) -or the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), -and compare the results from what you see in GitLab. - -If the results: - -- Sync up, then there is not a technical "issue." Instead, it might be a problem - with the Elasticsearch filters we are using. This can be complicated, so it is best to - escalate to GitLab support to check these and guide you on the potential on whether or - not a feature request is needed. -- Do not match up, this indicates a problem with the documents generated from the - project. It is best to re-index that project and proceed with - [Troubleshooting indexing](#troubleshooting-indexing). - -### Troubleshooting indexing - -Troubleshooting indexing issues can be tricky. It can pretty quickly go to either GitLab -support or your Elasticsearch administrator. - -The best place to start is to determine if the issue is with creating an empty index. -If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the -name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch -side and attempt to recreate it from the -[`recreate_index`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) -Rake task. - -If you still encounter issues, try creating an index manually on the Elasticsearch -instance. The details of the index aren't important here, as we want to test if indices -can be made. If the indices: - -- Cannot be made, speak with your Elasticsearch administrator. -- Can be made, Escalate this to GitLab support. - -If the issue is not with creating an empty index, the next step is to check for errors -during the indexing of projects. If errors do occur, they stem from either the indexing: - -- On the GitLab side. You need to rectify those. If they are not - something you are familiar with, contact GitLab support for guidance. -- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, speak with your Elasticsearch administrator. - -If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks: - -- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) -- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) - -If: - -- Everything is showing at 100%, escalate to GitLab support. This could be a potential - bug/issue. -- You do see something not at 100%, attempt to reindex that project. To do this, - run `sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=<project ID> ID_TO=<project ID>`. - -If reindexing the project shows: - -- Errors on the GitLab side, escalate those to GitLab support. -- Elasticsearch errors or doesn't present any errors at all, reach out to your - Elasticsearch administrator to check the instance. - -### Troubleshooting integration - -Troubleshooting integration tends to be pretty straight forward, as there really isn't -much to "integrate" here. - -If the issue is: - -- With the Go indexer, check if the ICU development package is installed. - This is a required package so make sure you install it. - Go indexer was a beta indexer which can be optionally turned on/off, but in 12.3 it reached stable status and is now the default. -- Not concerning the Go indexer, it is almost always an - Elasticsearch-side issue. This means you should reach out to your Elasticsearch administrator - regarding the errors you are seeing. If you are unsure here, it never hurts to reach - out to GitLab support. - -Beyond that, review the error. If it is: - -- Specifically from the indexer, this could be a bug/issue and should be escalated to - GitLab support. -- An OS issue, you should reach out to your systems administrator. -- A `Faraday::TimeoutError (execution expired)` error **and** you're using a proxy, - [set a custom `gitlab_rails['env']` environment variable, called `no_proxy`](https://docs.gitlab.com/omnibus/settings/environment-variables.html) - with the IP address of your Elasticsearch host. - -### Troubleshooting performance - -Troubleshooting performance can be difficult on Elasticsearch. There is a ton of tuning -that *can* be done, but the majority of this falls on shoulders of a skilled -Elasticsearch administrator. - -Generally speaking, ensure: - -- The Elasticsearch server **is not** running on the same node as GitLab. -- The Elasticsearch server have enough RAM and CPU cores. -- That sharding **is** being used. - -Going into some more detail here, if Elasticsearch is running on the same server as GitLab, resource contention is **very** likely to occur. Ideally, Elasticsearch, which requires ample resources, should be running on its own server (maybe coupled with Logstash and Kibana). - -When it comes to Elasticsearch, RAM is the key resource. Elasticsearch themselves recommend: - -- **At least** 8 GB of RAM for a non-production instance. -- **At least** 16 GB of RAM for a production instance. -- Ideally, 64 GB of RAM. - -For CPU, Elasticsearch recommends at least 2 CPU cores, but Elasticsearch states common -setups use up to 8 cores. For more details on server specs, check out -[Elasticsearch's hardware guide](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html). - -Beyond the obvious, sharding comes into play. Sharding is a core part of Elasticsearch. -It allows for horizontal scaling of indices, which is helpful when you are dealing with -a large amount of data. - -With the way GitLab does indexing, there is a **huge** amount of documents being -indexed. By utilizing sharding, you can speed up Elasticsearch's ability to locate -data, since each shard is a Lucene index. - -If you are not using sharding, you are likely to hit issues when you start using -Elasticsearch in a production environment. - -Keep in mind that an index with only one shard has **no scale factor** and will -likely encounter issues when called upon with some frequency. - -If you need to know how many shards, read -[Elasticsearch's documentation on capacity planning](https://www.elastic.co/guide/en/elasticsearch/guide/2.x/capacity-planning.html), -as the answer is not straight forward. - -The easiest way to determine if sharding is in use is to check the output of the -[Elasticsearch Health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html): - -- Red means the cluster is down. -- Yellow means it is up with no sharding/replication. -- Green means it is healthy (up, sharding, replicating). - -For production use, it should always be green. - -Beyond these steps, you get into some of the more complicated things to check, -such as merges and caching. These can get complicated and it takes some time to -learn them, so it is best to escalate/pair with an Elasticsearch expert if you need to -dig further into these. - -Feel free to reach out to GitLab support, but this is likely to be something a skilled -Elasticsearch administrator has more experience with. - -### Troubleshooting Advanced Search migrations - -Troubleshooting Advanced Search migration failures can be difficult and may -require contacting an Elasticsearch administrator or GitLab Support. - -The best place to start while debugging issues with an Advanced Search -migration is the [`elasticsearch.log` file](../logs.md#elasticsearchlog). -Migrations log information while a migration is in progress and any -errors encountered. Apply fixes for any errors found in the log and retry -the migration. - -If you still encounter issues after retrying the migration, reach out to GitLab support. - -## Common issues - -All common issues [should be documented](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, -feel free to update that page with issues you encounter and solutions. - -## Replication - -Setting up Elasticsearch isn't too bad, but it can be a bit finicky and time consuming. - -The easiest method is to spin up a Docker container with the required version and -bind ports 9200/9300 so it can be used. - -The following is an example of running a Docker container of Elasticsearch v7.2.0: - -```shell -docker pull docker.elastic.co/elasticsearch/elasticsearch:7.2.0 -docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.2.0 -``` - -From here, you can: - -- Grab the IP of the Docker container (use `docker inspect <container_id>`) -- Use `<IP.add.re.ss:9200>` to communicate with it. - -This is a quick method to test out Elasticsearch, but by no means is this a -production solution. +<!-- This redirect file can be deleted after <2022-11-02>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 0ff1afa86ed..aa4dbec4f95 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -178,26 +178,6 @@ Feature.all Feature.all.map {|f| [f.name, f.state]} ``` -## Command Line - -### Check the GitLab version fast - -```shell -grep -m 1 gitlab /opt/gitlab/version-manifest.txt -``` - -### Debugging SSH - -```shell -GIT_SSH_COMMAND="ssh -vvv" git clone <repository> -``` - -### Debugging over HTTPS - -```shell -GIT_CURL_VERBOSE=1 GIT_TRACE=1 git clone <repository> -``` - ## Projects ### Clear a project's cache @@ -326,7 +306,7 @@ Project.find_each do |p| end ``` -## Bulk update to change all the Jira integrations to Jira instance-level values +### Bulk update to change all the Jira integrations to Jira instance-level values To change all Jira project to use the instance-level integration settings: @@ -341,6 +321,25 @@ To change all Jira project to use the instance-level integration settings: 1. Modify and save again the instance-level integration from the UI to propagate the changes to all the group-level and project-level integrations. +### Check if Jira Cloud is linked to a namespace + +```ruby +JiraConnectSubscription.where(namespace: Namespace.by_path('group/subgroup')) +``` + +### Check if Jira Cloud is linked to a project + +```ruby +Project.find_by_full_path('path/to/project').jira_subscription_exists? +``` + +### Check if Jira Cloud URL is linked to any namespace + +```ruby +installation = JiraConnectInstallation.find_by_base_url("https://customer_name.atlassian.net") +installation.subscriptions +``` + ### Bulk update to disable the Slack Notification service To disable notifications for all projects that have Slack service enabled, do: @@ -424,33 +423,6 @@ projects = Project.find_by_sql("SELECT * FROM projects WHERE name LIKE '%ject'") => [#<Project id:12 root/my-first-project>>, #<Project id:13 root/my-second-project>>] ``` -## Wikis - -### Recreate - -WARNING: -This is a destructive operation, the Wiki becomes empty. - -A Projects Wiki can be recreated by this command: - -```ruby -p = Project.find_by_full_path('<username-or-group>/<project-name>') ### enter your projects path - -GitlabShellWorker.perform_in(0, :remove_repository, p.repository_storage, p.wiki.disk_path) ### deletes the wiki project from the filesystem - -p.create_wiki ### creates the wiki project on the filesystem -``` - -## Issue boards - -### In case of issue boards not loading properly and it's getting time out. Call the Issue Rebalancing service to fix this - -```ruby -p = Project.find_by_full_path('<username-or-group>/<project-name>') - -Issues::RelativePositionRebalancingService.new(p.root_namespace.all_projects).execute -``` - ## Imports and exports ### Import a project @@ -855,52 +827,6 @@ Gitlab::CurrentSettings.update!(password_authentication_enabled_for_web: true) ## SCIM -### Fixing bad SCIM identities - -```ruby -def delete_bad_scim(email, group_path) - output = "" - u = User.find_by_email(email) - uid = u.id - g = Group.find_by_full_path(group_path) - saml_prov_id = SamlProvider.find_by(group_id: g.id).id - saml = Identity.where(user_id: uid, saml_provider_id: saml_prov_id) - scim = ScimIdentity.where(user_id: uid , group_id: g.id) - if saml[0] - saml_eid = saml[0].extern_uid - output += "%s," % [email] - output += "SAML: %s," % [saml_eid] - if scim[0] - scim_eid = scim[0].extern_uid - output += "SCIM: %s" % [scim_eid] - if saml_eid == scim_eid - output += " Identities matched, not deleted \n" - else - scim[0].destroy - output += " Deleted \n" - end - else - output = "ERROR No SCIM identify found for: [%s]\n" % [email] - puts output - return 1 - end - else - output = "ERROR No SAML identify found for: [%s]\n" % [email] - puts output - return 1 - end - puts output - return 0 -end - -# In case of multiple emails -emails = [email1, email2] - -emails.each do |e| - delete_bad_scim(e,'<group-path>') -end -``` - ### Find groups using an SQL query Find and store an array of groups based on an SQL query: @@ -927,13 +853,13 @@ conflicting_permanent_redirects.destroy_all ## Merge requests -### Close a merge request properly (if merged but still marked as open) +### Close a merge request ```ruby u = User.find_by_username('<username>') p = Project.find_by_full_path('<namespace/project>') m = p.merge_requests.find_by(iid: <iid>) -MergeRequests::PostMergeService.new(project: p, current_user: u).execute(m) +MergeRequests::CloseService.new(project: p, current_user: u).execute(m) ``` ### Delete a merge request @@ -954,6 +880,21 @@ m = p.merge_requests.find_by(iid: <iid>) MergeRequests::RebaseService.new(project: m.target_project, current_user: u).execute(m) ``` +### Set a merge request as merged + +Use when a merge request was accepted and the changes merged into the Git repository, +but the merge request still shows as open. + +If the changes are not merged yet, this action causes the merge request to +incorrectly show `merged into <branch-name>`. + +```ruby +u = User.find_by_username('<username>') +p = Project.find_by_full_path('<namespace/project>') +m = p.merge_requests.find_by(iid: <iid>) +MergeRequests::PostMergeService.new(project: p, current_user: u).execute(m) +``` + ## CI ### Cancel stuck pending pipelines @@ -1063,6 +1004,9 @@ License.current.trial? # License ID for lookup on CustomersDot License.current.license_id + +# License data in Base64-encoded ASCII format +License.current.data ``` ### Check if a project feature is available on the instance @@ -1378,16 +1322,6 @@ registry = Geo::SnippetRepositoryRegistry.find(registry_id) registry.replicator.send(:sync_repository) ``` -## Gitaly - -### Find available and used space - -A Gitaly storage resource can be polled through Rails to determine the available and used space. - -```ruby -Gitlab::GitalyClient::ServerService.new("default").storage_disk_statistics -``` - ## Generate Service Ping The [Service Ping Guide](../../development/service_ping/index.md) in our developer documentation @@ -1429,28 +1363,6 @@ Prints the metrics saved in `conversational_development_index_metrics`. rake gitlab:usage_data:generate_and_send ``` -## Kubernetes integration - -Find cluster: - -```ruby -cluster = Clusters::Cluster.find(1) -cluster = Clusters::Cluster.find_by(name: 'cluster_name') -``` - -Delete cluster without associated resources: - -```ruby -# Find users with the administrator access -user = User.find_by(username: 'admin_user') - -# Find the cluster with the ID -cluster = Clusters::Cluster.find(1) - -# Delete the cluster -Clusters::DestroyService.new(user).execute(cluster) -``` - ## Elasticsearch ### Configuration attributes diff --git a/doc/administration/troubleshooting/group_saml_scim.md b/doc/administration/troubleshooting/group_saml_scim.md index 145eb5f65ae..b5187504231 100644 --- a/doc/administration/troubleshooting/group_saml_scim.md +++ b/doc/administration/troubleshooting/group_saml_scim.md @@ -1,207 +1,11 @@ --- -stage: Manage -group: Authentication and Authorization -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -type: reference +redirect_to: '../../user/group/saml_sso/example_saml_config.md' +remove_date: '2022-10-29' --- -# Troubleshooting Group SAML and SCIM **(PREMIUM SAAS)** +This document was moved to [another location](../../user/group/saml_sso/example_saml_config.md). -These are notes and screenshots regarding Group SAML and SCIM that the GitLab Support Team sometimes uses while troubleshooting, but which do not fit into the official documentation. GitLab is making this public, so that anyone can make use of the Support team's collected knowledge. - -Please refer to the GitLab [Group SAML](../../user/group/saml_sso/index.md) docs for information on the feature and how to set it up. - -When troubleshooting a SAML configuration, GitLab team members will frequently start with the [SAML troubleshooting section](../../user/group/saml_sso/index.md#troubleshooting). - -They may then set up a test configuration of the desired identity provider. We include example screenshots in this section. - -## SAML and SCIM screenshots - -This section includes relevant screenshots of the following example configurations of [Group SAML](../../user/group/saml_sso/index.md) and [Group SCIM](../../user/group/saml_sso/scim_setup.md): - -- [Azure Active Directory](#azure-active-directory) -- [Google Workspace](#google-workspace) -- [Okta](#okta) -- [OneLogin](#onelogin) - -WARNING: -These screenshots are updated only as needed by GitLab Support. They are **not** official documentation. - -If you are currently having an issue with GitLab, you may want to check your [support options](https://about.gitlab.com/support/). - -## Azure Active Directory - -Basic SAML app configuration: - - - -User claims and attributes: - - - -SCIM mapping: - - - - -Group Sync: - - - -## Google Workspace - -Basic SAML app configuration: - - - -User claims and attributes: - - - -IdP links and certificate: - -NOTE: -Google Workspace displays a SHA256 fingerprint. To retrieve the SHA1 fingerprint required by GitLab for configuring SAML, download the certificate and calculate the SHA1 certificate -fingerprint. - - - -## Okta - -Basic SAML app configuration: - - - -User claims and attributes: - - - -Advanced SAML app settings (defaults): - - - -IdP Links and Certificate: - - - -Sign on settings: - - - -Self-managed instance example: - - - -Setting the username for the newly provisioned users when assigning them the SCIM app: - - - -## OneLogin - -Application details: - - - -Parameters: - - - -Adding a user: - - - -SSO settings: - - - -## SAML response example - -When a user signs in using SAML, GitLab receives a SAML response. The SAML response can be found in `production.log` logs as a base64-encoded message. Locate the response by -searching for `SAMLResponse`. The decoded SAML response is in XML format. For example: - -```xml -<?xml version="1.0" encoding="UTF-8"?> -<saml2p:Response xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" xmlns:xs="http://www.w3.org/2001/XMLSchema" Destination="https://gitlabexample/-/saml/callback" ID="id4898983630840142426821432" InResponseTo="_c65e4c88-9425-4472-b42c-37f4186ac0ee" IssueInstant="2022-05-30T21:30:35.696Z" Version="2.0"> - <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" Format="urn:oasis:names:tc:SAML:2.0:nameid-format:entity">http://www.okta.com/exk2y6j57o1Pdr2lI8qh7</saml2:Issuer> - <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"> - <ds:SignedInfo> - <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/> - <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/> - <ds:Reference URI="#id4898983630840142426821432"> - <ds:Transforms> - <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/> - <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> - <ec:InclusiveNamespaces xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="xs"/> - </ds:Transform> - </ds:Transforms> - <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/> - <ds:DigestValue>neiQvv9d3OgS4GZW8Nptp4JhjpKs3GCefibn+vmRgk4=</ds:DigestValue> - </ds:Reference> - </ds:SignedInfo> - <ds:SignatureValue>dMsQX8ivi...HMuKGhyLRvabGU6CuPrf7==</ds:SignatureValue> - <ds:KeyInfo> - <ds:X509Data> - <ds:X509Certificate>MIIDq...cptGr3vN9TQ==</ds:X509Certificate> - </ds:X509Data> - </ds:KeyInfo> - </ds:Signature> - <saml2p:Status xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol"> - <saml2p:StatusCode Value="urn:oasis:names:tc:SAML:2.0:status:Success"/> - </saml2p:Status> - <saml2:Assertion xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" xmlns:xs="http://www.w3.org/2001/XMLSchema" ID="id489" IssueInstant="2022-05-30T21:30:35.696Z" Version="2.0"> - <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" Format="urn:oasis:names:tc:SAML:2.0:nameid-format:entity">http://www.okta.com/exk2y6j57o1Pdr2lI8qh7</saml2:Issuer> - <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"> - <ds:SignedInfo> - <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/> - <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/> - <ds:Reference URI="#id48989836309833801859473359"> - <ds:Transforms> - <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/> - <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> - <ec:InclusiveNamespaces xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="xs"/> - </ds:Transform> - </ds:Transforms> - <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/> - <ds:DigestValue>MaIsoi8hbT9gsi/mNZsz449mUuAcuEWY0q3bc4asOQs=</ds:DigestValue> - </ds:Reference> - </ds:SignedInfo> - <ds:SignatureValue>dMsQX8ivi...HMuKGhyLRvabGU6CuPrf7==<</ds:SignatureValue> - <ds:KeyInfo> - <ds:X509Data> - <ds:X509Certificate>MIIDq...cptGr3vN9TQ==</ds:X509Certificate> - </ds:X509Data> - </ds:KeyInfo> - </ds:Signature> - <saml2:Subject xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion"> - <saml2:NameID Format="urn:oasis:names:tc:SAML:2.0:nameid-format:persistent">useremail@domain.com</saml2:NameID> - <saml2:SubjectConfirmation Method="urn:oasis:names:tc:SAML:2.0:cm:bearer"> - <saml2:SubjectConfirmationData InResponseTo="_c65e4c88-9425-4472-b42c-37f4186ac0ee" NotOnOrAfter="2022-05-30T21:35:35.696Z" Recipient="https://gitlab.example.com/-/saml/callback"/> - </saml2:SubjectConfirmation> - </saml2:Subject> - <saml2:Conditions xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" NotBefore="2022-05-30T21:25:35.696Z" NotOnOrAfter="2022-05-30T21:35:35.696Z"> - <saml2:AudienceRestriction> - <saml2:Audience>https://gitlab.example.com/</saml2:Audience> - </saml2:AudienceRestriction> - </saml2:Conditions> - <saml2:AuthnStatement xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" AuthnInstant="2022-05-30T21:30:35.696Z" SessionIndex="_c65e4c88-9425-4472-b42c-37f4186ac0ee"> - <saml2:AuthnContext> - <saml2:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml2:AuthnContextClassRef> - </saml2:AuthnContext> - </saml2:AuthnStatement> - <saml2:AttributeStatement xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion"> - <saml2:Attribute Name="email" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> - <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">useremail@domain.com</saml2:AttributeValue> - </saml2:Attribute> - <saml2:Attribute Name="firtname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> - <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">John</saml2:AttributeValue> - </saml2:Attribute> - <saml2:Attribute Name="lastname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> - <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">Doe</saml2:AttributeValue> - </saml2:Attribute> - <saml2:Attribute Name="Groups" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> - <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">Super-awesome-group</saml2:AttributeValue> - </saml2:Attribute> - </saml2:AttributeStatement> - </saml2:Assertion> -</saml2p:Response> -``` +<!-- This redirect file can be deleted after <2022-10-29>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/img/Okta-SAMLsetup.png b/doc/administration/troubleshooting/img/Okta-SAMLsetup.png Binary files differdeleted file mode 100644 index 1bd9bf4d7e9..00000000000 --- a/doc/administration/troubleshooting/img/Okta-SAMLsetup.png +++ /dev/null diff --git a/doc/administration/troubleshooting/img/Okta-advancedsettings.png b/doc/administration/troubleshooting/img/Okta-advancedsettings.png Binary files differdeleted file mode 100644 index 45e378d1d12..00000000000 --- a/doc/administration/troubleshooting/img/Okta-advancedsettings.png +++ /dev/null diff --git a/doc/administration/troubleshooting/img/Okta-attributes.png b/doc/administration/troubleshooting/img/Okta-attributes.png Binary files differdeleted file mode 100644 index a3405e4de9b..00000000000 --- a/doc/administration/troubleshooting/img/Okta-attributes.png +++ /dev/null diff --git a/doc/administration/troubleshooting/img/Okta-linkscert.png b/doc/administration/troubleshooting/img/Okta-linkscert.png Binary files differdeleted file mode 100644 index 38cae415f7e..00000000000 --- a/doc/administration/troubleshooting/img/Okta-linkscert.png +++ /dev/null diff --git a/doc/administration/troubleshooting/img/okta_admin_panel_v13_9.png b/doc/administration/troubleshooting/img/okta_admin_panel_v13_9.png Binary files differdeleted file mode 100644 index 2ebb1f0112c..00000000000 --- a/doc/administration/troubleshooting/img/okta_admin_panel_v13_9.png +++ /dev/null diff --git a/doc/administration/troubleshooting/img/okta_saml_settings.png b/doc/administration/troubleshooting/img/okta_saml_settings.png Binary files differdeleted file mode 100644 index ee275ece369..00000000000 --- a/doc/administration/troubleshooting/img/okta_saml_settings.png +++ /dev/null diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md index 7fe731bda66..429dc79e95f 100644 --- a/doc/administration/troubleshooting/index.md +++ b/doc/administration/troubleshooting/index.md @@ -13,15 +13,12 @@ installation. - [SSL](ssl.md) - [Geo](../geo/replication/troubleshooting.md) -- [Elasticsearch](elasticsearch.md) -- [Sidekiq](sidekiq.md) - [GitLab Rails console cheat sheet](gitlab_rails_cheat_sheet.md) -- [Group SAML and SCIM troubleshooting](group_saml_scim.md) **(PREMIUM SAAS)** +- [Example group SAML and SCIM configurations](../../user/group/saml_sso/example_saml_config.md) - [Kubernetes cheat sheet](https://docs.gitlab.com/charts/troubleshooting/kubernetes_cheat_sheet.html) - [Linux cheat sheet](linux_cheat_sheet.md) -- [Parsing GitLab logs with `jq`](log_parsing.md) +- [Parsing GitLab logs with `jq`](../logs/log_parsing.md) - [Diagnostics tools](diagnostics_tools.md) -- [Tracing requests with correlation ID](tracing_correlation_id.md) Some feature documentation pages also have a troubleshooting section at the end that you can check for feature-specific help. diff --git a/doc/administration/troubleshooting/linux_cheat_sheet.md b/doc/administration/troubleshooting/linux_cheat_sheet.md index 66d5fb82936..6ff6e562a7d 100644 --- a/doc/administration/troubleshooting/linux_cheat_sheet.md +++ b/doc/administration/troubleshooting/linux_cheat_sheet.md @@ -14,7 +14,7 @@ having an issue with GitLab, you may want to check your [support options](https: first, before attempting to use this information. WARNING: -It is [beyond the scope of GitLab Support to assist in systems administration](https://about.gitlab.com/support/statement-of-support.html#training). GitLab administrators are expected to know these commands for their distribution +It is [beyond the scope of GitLab Support to assist in systems administration](https://about.gitlab.com/support/statement-of-support/#training). GitLab administrators are expected to know these commands for their distribution of choice. If you are a GitLab Support Engineer, consider this a cross-reference to translate `yum` -> `apt-get` and the like. @@ -204,24 +204,39 @@ or you can build it from source if you have the Rust compiler. #### How to use the tool -First run the tool with no arguments other than the strace output filename to get -a summary of the top processes sorted by time spent actively performing tasks. You -can also sort based on total time, # of system calls made, PID #, and # of child processes -using the `-S` or `--sort` flag. The number of results defaults to 25 processes, but +First run the tool with `summary` flag to get a summary of the top processes sorted by time spent actively performing tasks. +You can also sort based on total time, # of system calls made, PID #, and # of child processes +using the `-s` or `--sort` flag. The number of results defaults to 25 processes, but can be changed using the `-c`/`--count` option. See `--help` for full details. ```shell -$ ./strace-parser strace.txt +$ ./strace-parser sidekiq_trace.txt summary -c15 -s=pid -Top 25 PIDs +Top 15 PIDs by PID # ----------- - pid active (ms) wait (ms) total (ms) % active syscalls - ---------- ---------- --------- --------- --------- --------- - 8795 689.072 45773.832 46462.902 16.89% 23018 - 13408 679.432 55910.891 56590.320 16.65% 28593 - 6423 554.822 13175.485 13730.308 13.60% 13735 -... + pid actv (ms) wait (ms) user (ms) total (ms) % of actv syscalls children + ------- ---------- ---------- ---------- ---------- --------- --------- --------- + 16706 0.000 0.000 0.000 0.000 0.00% 0 0 + 16708 0.000 0.000 0.000 0.000 0.00% 0 0 + 16716 0.000 0.000 0.000 0.000 0.00% 0 0 + 16717 0.000 0.000 0.000 0.000 0.00% 0 0 + 16718 0.000 0.000 0.000 0.000 0.00% 0 0 + 16719 0.000 0.000 0.000 0.000 0.00% 0 0 + 16720 0.389 9796.434 1.090 9797.912 0.02% 16 0 + 16721 0.000 0.000 0.000 0.000 0.00% 0 0 + 16722 0.000 0.000 0.000 0.000 0.00% 0 0 + 16723 0.000 0.000 0.000 0.000 0.00% 0 0 + 16804 0.218 11099.535 1.881 11101.634 0.01% 36 0 + 16813 0.000 0.000 0.000 0.000 0.00% 0 0 + 16814 1.740 11825.640 4.616 11831.996 0.10% 57 0 + 16815 2.364 12039.993 7.669 12050.026 0.14% 80 0 + 16816 0.000 0.000 0.000 0.000 0.00% 0 0 + +PIDs 93 +real 0m12.287s +user 0m1.474s +sys 0m1.686s ``` Based on the summary, you can then view the details of system calls made by one or more @@ -229,36 +244,38 @@ processes using the `-p`/`--pid` for a specific process, or `-s`/`--stats` flags a sorted list. `--stats` takes the same sorting and count options as summary. ```shell -$ ./strace-parse strace.text -p 6423 - -PID 6423 -13735 syscalls, active time: 554.822ms, total time: 13730.308ms - - syscall count total max avg min errors - (ms) (ms) (ms) (ms) - --------------- -------- ---------- ---------- ---------- ---------- -------- - epoll_wait 628 13175.485 21.259 20.980 0.020 - clock_gettime 7326 199.500 0.249 0.027 0.013 - stat 2101 110.768 19.056 0.053 0.017 ENOENT: 2076 - ... +./strace-parser sidekiq_trace.txt p 16815 + +PID 16815 + + 80 syscalls, active time: 2.364ms, user time: 7.669ms, total time: 12050.026ms + start time: 22:46:14.830267 end time: 22:46:26.880293 + + syscall count total (ms) max (ms) avg (ms) min (ms) errors + ----------------- -------- ---------- ---------- ---------- ---------- -------- + futex 5 10100.229 5400.106 2020.046 0.022 ETIMEDOUT: 2 + restart_syscall 1 1939.764 1939.764 1939.764 1939.764 ETIMEDOUT: 1 + getpid 33 1.020 0.046 0.031 0.018 + clock_gettime 14 0.420 0.038 0.030 0.021 + stat 6 0.277 0.072 0.046 0.031 + read 6 0.170 0.036 0.028 0.020 + openat 3 0.126 0.045 0.042 0.038 + close 3 0.099 0.034 0.033 0.031 + lseek 3 0.089 0.035 0.030 0.021 + ioctl 3 0.082 0.033 0.027 0.023 ENOTTY: 3 + fstat 3 0.081 0.034 0.027 0.022 --------------- - Parent PID: 495 - Child PIDs: 8383, 8418, 8419, 8420, 8421 + Slowest file open times for PID 16815: - Slowest file access times for PID 6423: - - open (ms) timestamp error file name - ----------- --------------- --------------- ---------- - 29.818 10:53:11.528954 /srv/gitlab-data/builds/2018_08/6174/954448.log - 12.309 10:53:46.708274 /srv/gitlab-data/builds/2018_08/5342/954186.log - 0.039 10:53:49.222110 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_note.html.haml - 0.035 10:53:49.125115 /opt/gitlab/embedded/service/gitlab-rails/app/views/events/event/_push.html.haml - ... + dur (ms) timestamp error file name + ---------- --------------- --------------- --------- + 0.045 22:46:16.771318 - /opt/gitlab/embedded/service/gitlab-rails/config/database.yml + 0.043 22:46:26.877954 - /opt/gitlab/embedded/service/gitlab-rails/config/database.yml + 0.038 22:46:22.174610 - /opt/gitlab/embedded/service/gitlab-rails/config/database.yml ``` -In the example above, we can see that file opening times on `/srv/gitlab-data` are -extremely slow, about 100X slower than `/opt/gitlab`. +In the example above, we can see which files took longer to open for `PID 16815`. When nothing stands out in the results, a good way to get more context is to run `strace` on your own GitLab instance while performing the action performed by the customer, diff --git a/doc/administration/troubleshooting/log_parsing.md b/doc/administration/troubleshooting/log_parsing.md index 0320b2e52ce..929a49494be 100644 --- a/doc/administration/troubleshooting/log_parsing.md +++ b/doc/administration/troubleshooting/log_parsing.md @@ -1,316 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../logs/log_parsing.md' +remove_date: '2022-10-24' --- -# Parsing GitLab logs with `jq` **(FREE SELF)** +This document was moved to [another location](../logs/log_parsing.md). -We recommend using log aggregation and search tools like Kibana and Splunk whenever possible, -but if they are not available you can still quickly parse -[GitLab logs](../logs.md) in JSON format -(the default in GitLab 12.0 and later) using [`jq`](https://stedolan.github.io/jq/). - -NOTE: -Specifically for summarizing error events and basic usage statistics, -the GitLab Support Team provides the specialised -[`fast-stats` tool](https://gitlab.com/gitlab-com/support/toolbox/fast-stats/#when-to-use-it). - -## What is JQ? - -As noted in its [manual](https://stedolan.github.io/jq/manual/), `jq` is a command-line JSON processor. The following examples -include use cases targeted for parsing GitLab log files. - -## Parsing Logs - -The examples listed below address their respective log files by -their relative Omnibus paths and default filenames. -Find the respective full paths in the [GitLab logs sections](../logs.md#production_jsonlog). - -### General Commands - -#### Pipe colorized `jq` output into `less` - -```shell -jq . <FILE> -C | less -R -``` - -#### Search for a term and pretty-print all matching lines - -```shell -grep <TERM> <FILE> | jq . -``` - -#### Skip invalid lines of JSON - -```shell -jq -cR 'fromjson?' file.json | jq <COMMAND> -``` - -By default `jq` errors out when it encounters a line that is not valid JSON. -This skips over all invalid lines and parses the rest. - -#### Print a JSON log's time range - -```shell -cat log.json | (head -1; tail -1) | jq '.time' -``` - -Use `zcat` if the file has been rotated and compressed: - -```shell -zcat @400000006026b71d1a7af804.s | (head -1; tail -1) | jq '.time' - -zcat some_json.log.25.gz | (head -1; tail -1) | jq '.time' -``` - -#### Get activity for correlation ID across multiple JSON logs in chronological order - -```shell -grep -hR <correlationID> | jq -c -R 'fromjson?' | jq -C -s 'sort_by(.time)' | less -R -``` - -### Parsing `gitlab-rails/production_json.log` and `gitlab-rails/api_json.log` - -#### Find all requests with a 5XX status code - -```shell -jq 'select(.status >= 500)' <FILE> -``` - -#### Top 10 slowest requests - -```shell -jq -s 'sort_by(-.duration_s) | limit(10; .[])' <FILE> -``` - -#### Find and pretty print all requests related to a project - -```shell -grep <PROJECT_NAME> <FILE> | jq . -``` - -#### Find all requests with a total duration > 5 seconds - -```shell -jq 'select(.duration_s > 5000)' <FILE> -``` - -#### Find all project requests with more than 5 rugged calls - -```shell -grep <PROJECT_NAME> <FILE> | jq 'select(.rugged_calls > 5)' -``` - -#### Find all requests with a Gitaly duration > 10 seconds - -```shell -jq 'select(.gitaly_duration_s > 10000)' <FILE> -``` - -#### Find all requests with a queue duration > 10 seconds - -```shell -jq 'select(.queue_duration_s > 10000)' <FILE> -``` - -#### Top 10 requests by # of Gitaly calls - -```shell -jq -s 'map(select(.gitaly_calls != null)) | sort_by(-.gitaly_calls) | limit(10; .[])' <FILE> -``` - -### Parsing `gitlab-rails/production_json.log` - -#### Print the top three controller methods by request volume and their three longest durations - -```shell -jq -s -r 'group_by(.controller+.action) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tMETHOD: \(.[0].controller)#\(.[0].action)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' production_json.log -``` - -**Example output** - -```plaintext -CT: 2721 METHOD: SessionsController#new DURS: 844.06, 713.81, 704.66 -CT: 2435 METHOD: MetricsController#index DURS: 299.29, 284.01, 158.57 -CT: 1328 METHOD: Projects::NotesController#index DURS: 403.99, 386.29, 384.39 -``` - -### Parsing `gitlab-rails/api_json.log` - -#### Print top three routes with request count and their three longest durations - -```shell -jq -s -r 'group_by(.route) | sort_by(-length) | limit(3; .[]) | sort_by(-.duration_s) | "CT: \(length)\tROUTE: \(.[0].route)\tDURS: \(.[0].duration_s), \(.[1].duration_s), \(.[2].duration_s)"' api_json.log -``` - -**Example output** - -```plaintext -CT: 2472 ROUTE: /api/:version/internal/allowed DURS: 56402.65, 38411.43, 19500.41 -CT: 297 ROUTE: /api/:version/projects/:id/repository/tags DURS: 731.39, 685.57, 480.86 -CT: 190 ROUTE: /api/:version/projects/:id/repository/commits DURS: 1079.02, 979.68, 958.21 -``` - -### Print top API user agents - -```shell -jq --raw-output '[.route, .ua] | @tsv' api_json.log | sort | uniq -c | sort -n -``` - -**Example output**: - -```plaintext - 89 /api/:version/usage_data/increment_unique_users # plus browser details - 567 /api/:version/jobs/:id/trace gitlab-runner # plus version details -1234 /api/:version/internal/allowed GitLab-Shell -``` - -This sample response seems normal. A custom tool or script might be causing a high load -if the output contains many: - -- Third party libraries like `python-requests` or `curl`. -- [GitLab CLI clients](https://about.gitlab.com/partners/technology-partners/#cli-clients). - -You can also [use `fast-stats top`](#parsing-gitlab-logs-with-jq) to extract performance statistics. - -### Parsing `gitlab-workhorse/current` - -### Print top Workhorse user agents - -```shell -jq --raw-output '[.uri, .user_agent] | @tsv' current | sort | uniq -c | sort -n -``` - -**Example output**: - -```plaintext - 89 /api/graphql # plus browser details - 567 /api/v4/internal/allowed GitLab-Shell -1234 /api/v4/jobs/request gitlab-runner # plus version details -``` - -Similar to the [API `ua` data](#print-top-api-user-agents), -deviations from this common order might indicate scripts that could be optimized. - -The performance impact of runners checking for new jobs can be reduced by increasing -[the `check_interval` setting](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section), -for example. - -### Parsing `gitlab-rails/geo.log` - -#### Find most common Geo sync errors - -If [the `geo:status` Rake task](../geo/replication/troubleshooting.md#sync-status-rake-task) -repeatedly reports that some items never reach 100%, -the following command helps to focus on the most common errors. - -```shell -jq --raw-output 'select(.severity == "ERROR") | [.project_path, .message] | @tsv' geo.log | sort | uniq -c | sort | tail -``` - -### Parsing `gitaly/current` - -Use the following examples to [troubleshoot Gitaly](../gitaly/troubleshooting.md). - -#### Find all Gitaly requests sent from web UI - -```shell -jq 'select(."grpc.meta.client_name" == "gitlab-web")' current -``` - -#### Find all failed Gitaly requests - -```shell -jq 'select(."grpc.code" != null and ."grpc.code" != "OK")' current -``` - -#### Find all requests that took longer than 30 seconds - -```shell -jq 'select(."grpc.time_ms" > 30000)' current -``` - -#### Print top ten projects by request volume and their three longest durations - -```shell -jq --raw-output --slurp ' - map( - select( - ."grpc.request.glProjectPath" != null - and ."grpc.request.glProjectPath" != "" - and ."grpc.time_ms" != null - ) - ) - | group_by(."grpc.request.glProjectPath") - | sort_by(-length) - | limit(10; .[]) - | sort_by(-."grpc.time_ms") - | [ - length, - .[0]."grpc.time_ms", - .[1]."grpc.time_ms", - .[2]."grpc.time_ms", - .[0]."grpc.request.glProjectPath" - ] - | @sh' current \ -| awk 'BEGIN { printf "%7s %10s %10s %10s\t%s\n", "CT", "MAX DURS", "", "", "PROJECT" } - { printf "%7u %7u ms, %7u ms, %7u ms\t%s\n", $1, $2, $3, $4, $5 }' -``` - -**Example output** - -```plaintext - CT MAX DURS PROJECT - 206 4898 ms, 1101 ms, 1032 ms 'groupD/project4' - 109 1420 ms, 962 ms, 875 ms 'groupEF/project56' - 663 106 ms, 96 ms, 94 ms 'groupABC/project123' - ... -``` - -#### Find all projects affected by a fatal Git problem - -```shell -grep "fatal: " current | \ - jq '."grpc.request.glProjectPath"' | \ - sort | uniq -``` - -### Parsing `gitlab-shell/gitlab-shell.log` - -For investigating Git calls via SSH, from [GitLab 12.10](https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/367). - -Find the top 20 calls by project and user: - -```shell -jq --raw-output --slurp ' - map( - select( - .username != null and - .gl_project_path !=null - ) - ) - | group_by(.username+.gl_project_path) - | sort_by(-length) - | limit(20; .[]) - | "count: \(length)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ - gitlab-shell.log -``` - -Find the top 20 calls by project, user, and command: - -```shell -jq --raw-output --slurp ' - map( - select( - .command != null and - .username != null and - .gl_project_path !=null - ) - ) - | group_by(.username+.gl_project_path+.command) - | sort_by(-length) - | limit(20; .[]) - | "count: \(length)\tcommand: \(.[0].command)\tuser: \(.[0].username)\tproject: \(.[0].gl_project_path)" ' \ - gitlab-shell.log -``` +<!-- This redirect file can be deleted after <2022-10-24>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/postgresql.md b/doc/administration/troubleshooting/postgresql.md index 61b661d45f8..9e3d3d47a10 100644 --- a/doc/administration/troubleshooting/postgresql.md +++ b/doc/administration/troubleshooting/postgresql.md @@ -216,7 +216,7 @@ because the statement timeout was too short: pg_dump: error: Error message from server: server closed the connection unexpectedly ``` -You may also see errors in the [PostgreSQL logs](../logs.md#postgresql-logs): +You may also see errors in the [PostgreSQL logs](../logs/index.md#postgresql-logs): ```plaintext canceling statement due to statement timeout diff --git a/doc/administration/troubleshooting/sidekiq.md b/doc/administration/troubleshooting/sidekiq.md index 40bfe22ac63..e49e0ed4f1c 100644 --- a/doc/administration/troubleshooting/sidekiq.md +++ b/doc/administration/troubleshooting/sidekiq.md @@ -1,395 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../sidekiq/sidekiq_troubleshooting.md' +remove_date: '2022-11-11' --- -# Troubleshooting Sidekiq **(FREE SELF)** +This document was moved to [another location](../sidekiq/sidekiq_troubleshooting.md). -Sidekiq is the background job processor GitLab uses to asynchronously run -tasks. When things go wrong it can be difficult to troubleshoot. These -situations also tend to be high-pressure because a production system job queue -may be filling up. Users will notice when this happens because new branches -may not show up and merge requests may not be updated. The following are some -troubleshooting steps to help you diagnose the bottleneck. - -GitLab administrators/users should consider working through these -debug steps with GitLab Support so the backtraces can be analyzed by our team. -It may reveal a bug or necessary improvement in GitLab. - -In any of the backtraces, be wary of suspecting cases where every -thread appears to be waiting in the database, Redis, or waiting to acquire -a mutex. This **may** mean there's contention in the database, for example, -but look for one thread that is different than the rest. This other thread -may be using all available CPU, or have a Ruby Global Interpreter Lock, -preventing other threads from continuing. - -## Log arguments to Sidekiq jobs - -[In GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44853) -some arguments passed to Sidekiq jobs are logged by default. -To avoid logging sensitive information (for instance, password reset tokens), -GitLab logs numeric arguments for all workers, with overrides for some specific -workers where their arguments are not sensitive. - -Example log output: - -```json -{"severity":"INFO","time":"2020-06-08T14:37:37.892Z","class":"AdminEmailsWorker","args":["[FILTERED]","[FILTERED]","[FILTERED]"],"retry":3,"queue":"admin_emails","backtrace":true,"jid":"9e35e2674ac7b12d123e13cc","created_at":"2020-06-08T14:37:37.373Z","meta.user":"root","meta.caller_id":"Admin::EmailsController#create","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:6dc94409cfdd4d77:9fbe19bdee865293:1","enqueued_at":"2020-06-08T14:37:37.410Z","pid":65011,"message":"AdminEmailsWorker JID-9e35e2674ac7b12d123e13cc: done: 0.48085 sec","job_status":"done","scheduling_latency_s":0.001012,"redis_calls":9,"redis_duration_s":0.004608,"redis_read_bytes":696,"redis_write_bytes":6141,"duration_s":0.48085,"cpu_s":0.308849,"completed_at":"2020-06-08T14:37:37.892Z","db_duration_s":0.010742} -{"severity":"INFO","time":"2020-06-08T14:37:37.894Z","class":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper","wrapped":"ActionMailer::MailDeliveryJob","queue":"mailers","args":["[FILTERED]"],"retry":3,"backtrace":true,"jid":"e47a4f6793d475378432e3c8","created_at":"2020-06-08T14:37:37.884Z","meta.user":"root","meta.caller_id":"AdminEmailsWorker","correlation_id":"37D3lArJmT1","uber-trace-id":"2d942cc98cc1b561:29344de0f966446d:5c3b0e0e1bef987b:1","enqueued_at":"2020-06-08T14:37:37.885Z","pid":65011,"message":"ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper JID-e47a4f6793d475378432e3c8: start","job_status":"start","scheduling_latency_s":0.009473} -{"severity":"INFO","time":"2020-06-08T14:39:50.648Z","class":"NewIssueWorker","args":["455","1"],"retry":3,"queue":"new_issue","backtrace":true,"jid":"a24af71f96fd129ec47f5d1e","created_at":"2020-06-08T14:39:50.643Z","meta.user":"root","meta.project":"h5bp/html5-boilerplate","meta.root_namespace":"h5bp","meta.caller_id":"Projects::IssuesController#create","correlation_id":"f9UCZHqhuP7","uber-trace-id":"28f65730f99f55a3:a5d2b62dec38dffc:48ddd092707fa1b7:1","enqueued_at":"2020-06-08T14:39:50.646Z","pid":65011,"message":"NewIssueWorker JID-a24af71f96fd129ec47f5d1e: start","job_status":"start","scheduling_latency_s":0.001144} -``` - -When using [Sidekiq JSON logging](../logs.md#sidekiqlog), -arguments logs are limited to a maximum size of 10 kilobytes of text; -any arguments after this limit are discarded and replaced with a -single argument containing the string `"..."`. - -You can set `SIDEKIQ_LOG_ARGUMENTS` [environment variable](https://docs.gitlab.com/omnibus/settings/environment-variables.html) -to `0` (false) to disable argument logging. - -Example: - -```ruby -gitlab_rails['env'] = {"SIDEKIQ_LOG_ARGUMENTS" => "0"} -``` - -In GitLab 13.5 and earlier, set `SIDEKIQ_LOG_ARGUMENTS` to `1` to start logging arguments passed to Sidekiq. - -## Thread dump - -Send the Sidekiq process ID the `TTIN` signal to output thread -backtraces in the log file. - -```shell -kill -TTIN <sidekiq_pid> -``` - -Check in `/var/log/gitlab/sidekiq/current` or `$GITLAB_HOME/log/sidekiq.log` for -the backtrace output. The backtraces are lengthy and generally start with -several `WARN` level messages. Here's an example of a single thread's backtrace: - -```plaintext -2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: ActiveRecord::RecordNotFound: Couldn't find Note with 'id'=3375386 -2016-04-13T06:21:20.022Z 31517 TID-orn4urby0 WARN: /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/activerecord-4.2.5.2/lib/active_record/core.rb:155:in `find' -/opt/gitlab/embedded/service/gitlab-rails/app/workers/new_note_worker.rb:7:in `perform' -/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:150:in `execute_job' -/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/processor.rb:132:in `block (2 levels) in process' -/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:127:in `block in invoke' -/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/memory_killer.rb:17:in `call' -/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/sidekiq-4.0.1/lib/sidekiq/middleware/chain.rb:129:in `block in invoke' -/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/sidekiq_middleware/arguments_logger.rb:6:in `call' -... -``` - -In some cases Sidekiq may be hung and unable to respond to the `TTIN` signal. -Move on to other troubleshooting methods if this happens. - -## Ruby profiling with `rbspy` - -[rbspy](https://rbspy.github.io) is an easy to use and low-overhead Ruby profiler that can be used to create -flamegraph-style diagrams of CPU usage by Ruby processes. - -No changes to GitLab are required to use it and it has no dependencies. To install it: - -1. Download the binary from the [`rbspy` releases page](https://github.com/rbspy/rbspy/releases). -1. Make the binary executable. - -To profile a Sidekiq worker for one minute, run: - -```shell -sudo ./rbspy record --pid <sidekiq_pid> --duration 60 --file /tmp/sidekiq_profile.svg -``` - - - -In this example of a flamegraph generated by `rbspy`, almost all of the Sidekiq process's time is spent in `rev_parse`, a native C -function in Rugged. In the stack, we can see `rev_parse` is being called by the `ExpirePipelineCacheWorker`. - -## Process profiling with `perf` - -Linux has a process profiling tool called `perf` that is helpful when a certain -process is eating up a lot of CPU. If you see high CPU usage and Sidekiq isn't -responding to the `TTIN` signal, this is a good next step. - -If `perf` is not installed on your system, install it with `apt-get` or `yum`: - -```shell -# Debian -sudo apt-get install linux-tools - -# Ubuntu (may require these additional Kernel packages) -sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` - -# Red Hat/CentOS -sudo yum install perf -``` - -Run `perf` against the Sidekiq PID: - -```shell -sudo perf record -p <sidekiq_pid> -``` - -Let this run for 30-60 seconds and then press Ctrl-C. Then view the `perf` report: - -```shell -$ sudo perf report - -# Sample output -Samples: 348K of event 'cycles', Event count (approx.): 280908431073 - 97.69% ruby nokogiri.so [.] xmlXPathNodeSetMergeAndClear - 0.18% ruby libruby.so.2.1.0 [.] objspace_malloc_increase - 0.12% ruby libc-2.12.so [.] _int_malloc - 0.10% ruby libc-2.12.so [.] _int_free -``` - -Above you see sample output from a `perf` report. It shows that 97% of the CPU is -being spent inside Nokogiri and `xmlXPathNodeSetMergeAndClear`. For something -this obvious you should then go investigate what job in GitLab would use -Nokogiri and XPath. Combine with `TTIN` or `gdb` output to show the -corresponding Ruby code where this is happening. - -## The GNU Project Debugger (`gdb`) - -`gdb` can be another effective tool for debugging Sidekiq. It gives you a little -more interactive way to look at each thread and see what's causing problems. - -Attaching to a process with `gdb` suspends the normal operation -of the process (Sidekiq does not process jobs while `gdb` is attached). - -Start by attaching to the Sidekiq PID: - -```shell -gdb -p <sidekiq_pid> -``` - -Then gather information on all the threads: - -```plaintext -info threads - -# Example output -30 Thread 0x7fe5fbd63700 (LWP 26060) 0x0000003f7cadf113 in poll () from /lib64/libc.so.6 -29 Thread 0x7fe5f2b3b700 (LWP 26533) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -28 Thread 0x7fe5f2a3a700 (LWP 26534) 0x0000003f7ce0ba5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -27 Thread 0x7fe5f2939700 (LWP 26535) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -26 Thread 0x7fe5f2838700 (LWP 26537) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -25 Thread 0x7fe5f2737700 (LWP 26538) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -24 Thread 0x7fe5f2535700 (LWP 26540) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -23 Thread 0x7fe5f2434700 (LWP 26541) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -22 Thread 0x7fe5f2232700 (LWP 26543) 0x0000003f7ce0b68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 -21 Thread 0x7fe5f2131700 (LWP 26544) 0x00007fe5f7b570f0 in xmlXPathNodeSetMergeAndClear () -from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -... -``` - -If you see a suspicious thread, like the Nokogiri one above, you may want -to get more information: - -```plaintext -thread 21 -bt - -# Example output -#0 0x00007ff0d6afe111 in xmlXPathNodeSetMergeAndClear () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#1 0x00007ff0d6b0b836 in xmlXPathNodeCollectAndTest () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#2 0x00007ff0d6b09037 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#3 0x00007ff0d6b09017 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#4 0x00007ff0d6b092e0 in xmlXPathCompOpEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#5 0x00007ff0d6b0bc37 in xmlXPathRunEval () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#6 0x00007ff0d6b0be5f in xmlXPathEvalExpression () from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/nokogiri-1.6.7.2/lib/nokogiri/nokogiri.so -#7 0x00007ff0d6a97dc3 in evaluate (argc=2, argv=0x1022d058, self=<value optimized out>) at xml_xpath_context.c:221 -#8 0x00007ff0daeab0ea in vm_call_cfunc_with_frame (th=0x1022a4f0, reg_cfp=0x1032b810, ci=<value optimized out>) at vm_insnhelper.c:1510 -``` - -To output a backtrace from all threads at once: - -```plaintext -set pagination off -thread apply all bt -``` - -Once you're done debugging with `gdb`, be sure to detach from the process and -exit: - -```plaintext -detach -exit -``` - -## Sidekiq kill signals - -TTIN was described above as the signal to print backtraces for logging, however -Sidekiq responds to other signals as well. For example, TSTP and TERM can be used -to gracefully shut Sidekiq down, see -[the Sidekiq Signals docs](https://github.com/mperham/sidekiq/wiki/Signals#ttin). - -## Check for blocking queries - -Sometimes the speed at which Sidekiq processes jobs can be so fast that it can -cause database contention. Check for blocking queries when backtraces above -show that many threads are stuck in the database adapter. - -The PostgreSQL wiki has details on the query you can run to see blocking -queries. The query is different based on PostgreSQL version. See -[Lock Monitoring](https://wiki.postgresql.org/wiki/Lock_Monitoring) for -the query details. - -## Managing Sidekiq queues - -It is possible to use [Sidekiq API](https://github.com/mperham/sidekiq/wiki/API) -to perform a number of troubleshooting steps on Sidekiq. - -These are the administrative commands and it should only be used if currently -administration interface is not suitable due to scale of installation. - -All these commands should be run using `gitlab-rails console`. - -### View the queue size - -```ruby -Sidekiq::Queue.new("pipeline_processing:build_queue").size -``` - -### Enumerate all enqueued jobs - -```ruby -queue = Sidekiq::Queue.new("chaos:chaos_sleep") -queue.each do |job| - # job.klass # => 'MyWorker' - # job.args # => [1, 2, 3] - # job.jid # => jid - # job.queue # => chaos:chaos_sleep - # job["retry"] # => 3 - # job.item # => { - # "class"=>"Chaos::SleepWorker", - # "args"=>[1000], - # "retry"=>3, - # "queue"=>"chaos:chaos_sleep", - # "backtrace"=>true, - # "queue_namespace"=>"chaos", - # "jid"=>"39bc482b823cceaf07213523", - # "created_at"=>1566317076.266069, - # "correlation_id"=>"c323b832-a857-4858-b695-672de6f0e1af", - # "enqueued_at"=>1566317076.26761}, - # } - - # job.delete if job.jid == 'abcdef1234567890' -end -``` - -### Enumerate currently running jobs - -```ruby -workers = Sidekiq::Workers.new -workers.each do |process_id, thread_id, work| - # process_id is a unique identifier per Sidekiq process - # thread_id is a unique identifier per thread - # work is a Hash which looks like: - # {"queue"=>"chaos:chaos_sleep", - # "payload"=> - # { "class"=>"Chaos::SleepWorker", - # "args"=>[1000], - # "retry"=>3, - # "queue"=>"chaos:chaos_sleep", - # "backtrace"=>true, - # "queue_namespace"=>"chaos", - # "jid"=>"b2a31e3eac7b1a99ff235869", - # "created_at"=>1566316974.9215662, - # "correlation_id"=>"e484fb26-7576-45f9-bf21-b99389e1c53c", - # "enqueued_at"=>1566316974.9229589}, - # "run_at"=>1566316974}], -end -``` - -### Remove Sidekiq jobs for given parameters (destructive) - -The general method to kill jobs conditionally is the following command, which -removes jobs that are queued but not started. Running jobs can not be killed. - -```ruby -queue = Sidekiq::Queue.new('<queue name>') -queue.each { |job| job.delete if <condition>} -``` - -Have a look at the section below for cancelling running jobs. - -In the method above, `<queue-name>` is the name of the queue that contains the jobs you want to delete and `<condition>` decides which jobs get deleted. - -Commonly, `<condition>` references the job arguments, which depend on the type of job in question. To find the arguments for a specific queue, you can have a look at the `perform` function of the related worker file, commonly found at `/app/workers/<queue-name>_worker.rb`. - -For example, `repository_import` has `project_id` as the job argument, while `update_merge_requests` has `project_id, user_id, oldrev, newrev, ref`. - -Arguments need to be referenced by their sequence ID using `job.args[<id>]` because `job.args` is a list of all arguments provided to the Sidekiq job. - -Here are some examples: - -```ruby -queue = Sidekiq::Queue.new('update_merge_requests') -# In this example, we want to remove any update_merge_requests jobs -# for the Project with ID 125 and ref `ref/heads/my_branch` -queue.each { |job| job.delete if job.args[0] == 125 and job.args[4] == 'ref/heads/my_branch' } -``` - -```ruby -# Cancelling jobs like: `RepositoryImportWorker.new.perform_async(100)` -id_list = [100] - -queue = Sidekiq::Queue.new('repository_import') -queue.each do |job| - job.delete if id_list.include?(job.args[0]) -end -``` - -### Remove specific job ID (destructive) - -```ruby -queue = Sidekiq::Queue.new('repository_import') -queue.each do |job| - job.delete if job.jid == 'my-job-id' -end -``` - -## Canceling running jobs (destructive) - -> Introduced in GitLab 12.3. - -This is highly risky operation and use it as last resort. -Doing that might result in data corruption, as the job -is interrupted mid-execution and it is not guaranteed -that proper rollback of transactions is implemented. - -```ruby -Gitlab::SidekiqDaemon::Monitor.cancel_job('job-id') -``` - -> This requires the Sidekiq to be run with `SIDEKIQ_MONITOR_WORKER=1` -> environment variable. - -To perform of the interrupt we use `Thread.raise` which -has number of drawbacks, as mentioned in [Why Ruby's Timeout is dangerous (and Thread.raise is terrifying)](https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/): - -> This is where the implications get interesting, and terrifying. This means that an exception can get raised: -> -> - during a network request (ok, as long as the surrounding code is prepared to catch Timeout::Error) -> - during the cleanup for the network request -> - during a rescue block -> - while creating an object to save to the database afterwards -> - in any of your code, regardless of whether it could have possibly raised an exception before -> -> Nobody writes code to defend against an exception being raised on literally any line. That's not even possible. So Thread.raise is basically like a sneak attack on your code that could result in almost anything. It would probably be okay if it were pure-functional code that did not modify any state. But this is Ruby, so that's unlikely :) - -## Disable Rugged - -Calls into Rugged, Ruby bindings for `libgit2`, [lock the Sidekiq processes's GVL](https://silverhammermba.github.io/emberb/c/#c-in-ruby-threads), -blocking all jobs on that worker from proceeding. If Rugged calls performed by Sidekiq are slow, this can cause significant delays in -background task processing. - -By default, Rugged is used when Git repository data is stored on local storage or on an NFS mount. -[Using Rugged is recommended when using NFS](../nfs.md#improving-nfs-performance-with-gitlab), but if -you are using local storage, disabling Rugged can improve Sidekiq performance: - -```shell -sudo gitlab-rake gitlab:features:disable_rugged -``` +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/administration/troubleshooting/tracing_correlation_id.md b/doc/administration/troubleshooting/tracing_correlation_id.md index 418dd729066..ee59b7c2504 100644 --- a/doc/administration/troubleshooting/tracing_correlation_id.md +++ b/doc/administration/troubleshooting/tracing_correlation_id.md @@ -1,202 +1,11 @@ --- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../logs/tracing_correlation_id.md' +remove_date: '2022-11-12' --- -# Finding relevant log entries with a correlation ID **(FREE SELF)** +This document was moved to [another location](../logs/tracing_correlation_id.md). -GitLab instances log a unique request tracking ID (known as the -"correlation ID") for most requests. Each individual request to GitLab gets -its own correlation ID, which then gets logged in each GitLab component's logs for that -request. This makes it easier to trace behavior in a -distributed system. Without this ID it can be difficult or -impossible to match correlating log entries. - -## Identify the correlation ID for a request - -The correlation ID is logged in structured logs under the key `correlation_id` -and in all response headers GitLab sends under the header `x-request-id`. -You can find your correlation ID by searching in either place. - -### Getting the correlation ID in your browser - -You can use your browser's developer tools to monitor and inspect network -activity with the site that you're visiting. See the links below for network monitoring -documentation for some popular browsers. - -- [Network Monitor - Firefox Developer Tools](https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor) -- [Inspect Network Activity In Chrome DevTools](https://developer.chrome.com/docs/devtools/network/) -- [Safari Web Development Tools](https://developer.apple.com/safari/tools/) -- [Microsoft Edge Network panel](https://docs.microsoft.com/en-us/microsoft-edge/devtools-guide-chromium/network/) - -To locate a relevant request and view its correlation ID: - -1. Enable persistent logging in your network monitor. Some actions in GitLab redirect you quickly after you submit a form, so this helps capture all relevant activity. -1. To help isolate the requests you are looking for, you can filter for `document` requests. -1. Select the request of interest to view further detail. -1. Go to the **Headers** section and look for **Response Headers**. There you should find an `x-request-id` header with a -value that was randomly generated by GitLab for the request. - -See the following example: - - - -### Getting the correlation ID from your logs - -Another approach to finding the correct correlation ID is to search or watch -your logs and find the `correlation_id` value for the log entry that you're -watching for. - -For example, let's say that you want learn what's happening or breaking when -you reproduce an action in GitLab. You could tail the GitLab logs, filtering -to requests by your user, and then watch the requests until you see what you're -interested in. - -### Getting the correlation ID from curl - -If you're using `curl` then you can use the verbose option to show request and response headers, as well as other debug information. - -```shell -➜ ~ curl --verbose "https://gitlab.example.com/api/v4/projects" -# look for a line that looks like this -< x-request-id: 4rAMkV3gof4 -``` - -#### Using jq - -This example uses [jq](https://stedolan.github.io/jq/) to filter results and -display values we most likely care about. - -```shell -sudo gitlab-ctl tail gitlab-rails/production_json.log | jq 'select(.username == "bob") | "User: \(.username), \(.method) \(.path), \(.controller)#\(.action), ID: \(.correlation_id)"' -``` - -```plaintext -"User: bob, GET /root/linux, ProjectsController#show, ID: U7k7fh6NpW3" -"User: bob, GET /root/linux/commits/master/signatures, Projects::CommitsController#signatures, ID: XPIHpctzEg1" -"User: bob, GET /root/linux/blob/master/README, Projects::BlobController#show, ID: LOt9hgi1TV4" -``` - -#### Using grep - -This example uses only `grep` and `tr`, which are more likely to be installed than `jq`. - -```shell -sudo gitlab-ctl tail gitlab-rails/production_json.log | grep '"username":"bob"' | tr ',' '\n' | egrep 'method|path|correlation_id' -``` - -```plaintext -{"method":"GET" -"path":"/root/linux" -"username":"bob" -"correlation_id":"U7k7fh6NpW3"} -{"method":"GET" -"path":"/root/linux/commits/master/signatures" -"username":"bob" -"correlation_id":"XPIHpctzEg1"} -{"method":"GET" -"path":"/root/linux/blob/master/README" -"username":"bob" -"correlation_id":"LOt9hgi1TV4"} -``` - -## Searching your logs for the correlation ID - -Once you have the correlation ID you can start searching for relevant log -entries. You can filter the lines by the correlation ID itself. -Combining a `find` and `grep` should be sufficient to find the entries you are looking for. - -```shell -# find <gitlab log directory> -type f -mtime -0 exec grep '<correlation ID>' '{}' '+' -find /var/log/gitlab -type f -mtime 0 -exec grep 'LOt9hgi1TV4' '{}' '+' -``` - -```plaintext -/var/log/gitlab/gitlab-workhorse/current:{"correlation_id":"LOt9hgi1TV4","duration_ms":2478,"host":"gitlab.domain.tld","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.tld/root/linux","remote_addr":"68.0.116.160:0","remote_ip":"[filtered]","status":200,"system":"http","time":"2019-09-17T22:17:19Z","uri":"/root/linux/blob/master/README?format=json\u0026viewer=rich","user_agent":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","written_bytes":1743} -/var/log/gitlab/gitaly/current:{"correlation_id":"LOt9hgi1TV4","grpc.code":"OK","grpc.meta.auth_version":"v2","grpc.meta.client_name":"gitlab-web","grpc.method":"FindCommits","grpc.request.deadline":"2019-09-17T22:17:47Z","grpc.request.fullMethod":"/gitaly.CommitService/FindCommits","grpc.request.glProjectPath":"root/linux","grpc.request.glRepository":"project-1","grpc.request.repoPath":"@hashed/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b.git","grpc.request.repoStorage":"default","grpc.request.topLevelGroup":"@hashed","grpc.service":"gitaly.CommitService","grpc.start_time":"2019-09-17T22:17:17Z","grpc.time_ms":2319.161,"level":"info","msg":"finished streaming call with code OK","peer.address":"@","span.kind":"server","system":"grpc","time":"2019-09-17T22:17:19Z"} -/var/log/gitlab/gitlab-rails/production_json.log:{"method":"GET","path":"/root/linux/blob/master/README","format":"json","controller":"Projects::BlobController","action":"show","status":200,"duration":2448.77,"view":0.49,"db":21.63,"time":"2019-09-17T22:17:19.800Z","params":[{"key":"viewer","value":"rich"},{"key":"namespace_id","value":"root"},{"key":"project_id","value":"linux"},{"key":"id","value":"master/README"}],"remote_ip":"[filtered]","user_id":2,"username":"bob","ua":"Mozilla/5.0 (Mac) Gecko Firefox/69.0","queue_duration":3.38,"gitaly_calls":1,"gitaly_duration":0.77,"rugged_calls":4,"rugged_duration_ms":28.74,"correlation_id":"LOt9hgi1TV4"} -``` - -### Searching in distributed architectures - -If you have done some horizontal scaling in your GitLab infrastructure, then -you must search across _all_ of your GitLab nodes. You can do this with -some sort of log aggregation software like Loki, ELK, Splunk, or others. - -You can use a tool like Ansible or PSSH (parallel SSH) that can execute identical commands across your servers in -parallel, or craft your own solution. - -### Viewing the request in the Performance Bar - -You can use the [performance bar](../monitoring/performance/performance_bar.md) to view interesting data including calls made to SQL and Gitaly. - -To view the data, the correlation ID of the request must match the same session as the user -viewing the performance bar. For API requests, this means that you must perform the request -using the session cookie of the signed-in user. - -For example, if you want to view the database queries executed for the following API endpoint: - -```shell -https://gitlab.com/api/v4/groups/2564205/projects?with_security_reports=true&page=1&per_page=1 -``` - -First, enable the **Developer Tools** panel. See [Getting the correlation ID in your browser](#getting-the-correlation-id-in-your-browser) for details on how to do this. - -After developer tools have been enabled, obtain a session cookie as follows: - -1. Visit <https://gitlab.com> while logged in. -1. Optional. Select **Fetch/XHR** request filter in the **Developer Tools** panel. This step is described for Google Chrome developer tools and is not strictly necessary, it just makes it easier to find the correct request. -1. Select the `results?request_id=<some-request-id>` request on the left hand side. -1. The session cookie is displayed under the `Request Headers` section of the `Headers` panel. Right-click on the cookie value and select `Copy value`. - - - -You have the value of the session cookie copied to your clipboard, for example: - -```shell -experimentation_subject_id=<subject-id>; _gitlab_session=<session-id>; event_filter=all; visitor_id=<visitor-id>; perf_bar_enabled=true; sidebar_collapsed=true; diff_view=inline; sast_entry_point_dismissed=true; auto_devops_settings_dismissed=true; cf_clearance=<cf-clearance>; collapsed_gutter=false; frequently_used_emojis=clap,thumbsup,rofl,tada,eyes,bow -``` - -Use the value of the session cookie to craft an API request by pasting it into a custom header of a `curl` request: - -```shell -$ curl --include "https://gitlab.com/api/v4/groups/2564205/projects?with_security_reports=true&page=1&per_page=1" \ ---header 'cookie: experimentation_subject_id=<subject-id>; _gitlab_session=<session-id>; event_filter=all; visitor_id=<visitor-id>; perf_bar_enabled=true; sidebar_collapsed=true; diff_view=inline; sast_entry_point_dismissed=true; auto_devops_settings_dismissed=true; cf_clearance=<cf-clearance>; collapsed_gutter=false; frequently_used_emojis=clap,thumbsup,rofl,tada,eyes,bow' - - date: Tue, 28 Sep 2021 03:55:33 GMT - content-type: application/json - ... - x-request-id: 01FGN8P881GF2E5J91JYA338Y3 - ... - [ - { - "id":27497069, - "description":"Analyzer for images used on live K8S containers based on Starboard" - }, - "container_registry_image_prefix":"registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning", - "..." - ] -``` - -The response contains the data from the API endpoint, and a `correlation_id` value, returned in the `x-request-id` header, as described in the [Identify the correlation ID for a request](#identify-the-correlation-id-for-a-request) section. - -You can then view the database details for this request: - -1. Paste the `x-request-id` value into the `request details` field of the [performance bar](../monitoring/performance/performance_bar.md) and press <kbd>Enter/Return</kbd>. This example uses the `x-request-id` value `01FGN8P881GF2E5J91JYA338Y3`, returned by the above response: - -  - -1. A new request is inserted into the `Request Selector` dropdown on the right-hand side of the Performance Bar. Select the new request to view the metrics of the API request: - -  - - <!-- vale gitlab.Substitutions = NO --> -1. Select the `pg` link in the Progress Bar to view the database queries executed by the API request: - -  - <!-- vale gitlab.Substitutions = YES --> - - The database query dialog is displayed: - -  +<!-- This redirect file can be deleted after 2022-11-12. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html -->
\ No newline at end of file diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 0bd46193d10..2dd8f1ed819 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -112,15 +112,15 @@ _The uploads are stored by default in `/home/git/gitlab/public/uploads`._ 1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: + lines, making sure to use the [appropriate ones for your provider](object_storage.md#connection-settings): ```yaml uploads: object_store: enabled: true remote_directory: "uploads" # The bucket name - connection: - provider: AWS # Only AWS supported at the moment + connection: # The lines in this block depend on your provider + provider: AWS aws_access_key_id: AWS_ACCESS_KEY_ID aws_secret_access_key: AWS_SECRET_ACCESS_KEY region: eu-central-1 diff --git a/doc/api/appearance.md b/doc/api/appearance.md index 7b98b226cde..7ad79fd66fe 100644 --- a/doc/api/appearance.md +++ b/doc/api/appearance.md @@ -104,7 +104,7 @@ PUT /application/appearance | Attribute | Type | Required | Description | | --------- | ------ | -------- | -------------- | -| `logo` | string | Yes | File to upload | +| `logo` | mixed | Yes | File to upload | Example request: diff --git a/doc/api/audit_events.md b/doc/api/audit_events.md index 753e01a15aa..80d7b23d642 100644 --- a/doc/api/audit_events.md +++ b/doc/api/audit_events.md @@ -137,12 +137,13 @@ Example response: The Group Audit Events API allows you to retrieve [group audit events](../administration/audit_events.md#group-events). This API cannot retrieve project audit events. -A user with a Owner role (or above) can retrieve group audit events of all users. -A user with a Developer or Maintainer role is limited to group audit events based on their individual actions. +A user with: -This endpoint optionally supports [keyset pagination](index.md#keyset-based-pagination): +- The Owner role can retrieve group audit events of all users. +- The Developer or Maintainer role is limited to group audit events based on their individual actions. -- When requesting consecutive pages of results, we recommend you use keyset pagination. +This endpoint supports both offset-based and [keyset-based](index.md#keyset-based-pagination) pagination. Keyset-based +pagination is recommended when requesting consecutive pages of results. ### Retrieve all group audit events diff --git a/doc/api/bulk_imports.md b/doc/api/bulk_imports.md index 8c5c4574206..913dc6ce4f1 100644 --- a/doc/api/bulk_imports.md +++ b/doc/api/bulk_imports.md @@ -27,7 +27,8 @@ POST /bulk_imports | `entities` | Array | yes | List of entities to import. | | `entities[source_type]` | String | yes | Source entity type (only `group_entity` is supported). | | `entities[source_full_path]` | String | yes | Source full path of the entity to import. | -| `entities[destination_name]` | String | yes | Destination slug for the entity. | +| `entities[destination_name]` | String | yes | Deprecated: Use :destination_slug instead. Destination slug for the entity. | +| `entities[destination_slug]` | String | yes | Destination slug for the entity. | | `entities[destination_namespace]` | String | no | Destination namespace for the entity. | ```shell @@ -41,7 +42,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla { "source_full_path": "source/full/path", "source_type": "group_entity", - "destination_name": "destination_slug", + "destination_slug": "destination_slug", "destination_namespace": "destination/namespace/path" } ] @@ -126,7 +127,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab "bulk_import_id": 1, "status": "finished", "source_full_path": "source_group", - "destination_name": "destination_slug", + "destination_slug": "destination_slug", "destination_namespace": "destination_path", "parent_id": null, "namespace_id": 1, @@ -140,7 +141,7 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab "bulk_import_id": 2, "status": "failed", "source_full_path": "another_group", - "destination_name": "another_slug", + "destination_slug": "another_slug", "destination_namespace": "another_namespace", "parent_id": null, "namespace_id": null, diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 6831f86e4ea..01fa4e11884 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -458,6 +458,37 @@ Deployments created by users on GitLab Premium or higher include the `approvals` } ``` +## Delete a specific deployment + +Delete a specific deployment that is not currently the last deployment for an environment or in a `running` state + +```plaintext +DELETE /projects/:id/deployments/:deployment_id +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `deployment_id` | integer | yes | The ID of the deployment | + +```shell +curl --request "DELETE" --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/deployments/1" +``` + +Example responses: + +```json +{ "message": "202 Accepted" } +``` + +```json +{ "message": "400 Cannot destroy running deployment" } +``` + +```json +{ "message": "400 Deployment currently deployed to environment" } +``` + ## List of merge requests associated with a deployment > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35739) in GitLab 12.7. diff --git a/doc/api/discussions.md b/doc/api/discussions.md index c30c00cef67..a5610adad79 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -170,7 +170,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing issue thread -Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). **WARNING** Notes can be added to other items than comments, such as system notes, making them threads. @@ -599,7 +599,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing epic thread Adds a new note to the thread. This can also -[create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +[create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). ```plaintext POST /groups/:id/epics/:epic_id/discussions/:discussion_id/notes @@ -1021,7 +1021,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>"\ ### Add note to existing merge request thread Adds a new note to the thread. This can also -[create a thread from a single comment](../user/discussions/#create-a-thread-by-replying-to-a-standard-comment). +[create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). ```plaintext POST /projects/:id/merge_requests/:merge_request_iid/discussions/:discussion_id/notes @@ -1103,7 +1103,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>"\ Gets a list of all discussion items for a single commit. ```plaintext -GET /projects/:id/commits/:commit_id/discussions +GET /projects/:id/repository/commits/:commit_id/discussions ``` | Attribute | Type | Required | Description | @@ -1237,7 +1237,7 @@ Diff comments contain also position: ```shell curl --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions" ``` ### Get single commit discussion item @@ -1245,7 +1245,7 @@ curl --header "PRIVATE-TOKEN: <your_access_token>"\ Returns a single discussion item for a specific project commit ```plaintext -GET /projects/:id/commits/:commit_id/discussions/:discussion_id +GET /projects/:id/repository/commits/:commit_id/discussions/:discussion_id ``` Parameters: @@ -1258,7 +1258,7 @@ Parameters: ```shell curl --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/<discussion_id>" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions/<discussion_id>" ``` ### Create new commit thread @@ -1267,7 +1267,7 @@ Creates a new thread to a single project commit. This is similar to creating a note but other comments (replies) can be added to it later. ```plaintext -POST /projects/:id/commits/:commit_id/discussions +POST /projects/:id/repository/commits/:commit_id/discussions ``` Parameters: @@ -1294,7 +1294,7 @@ Parameters: ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions?body=comment" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions?body=comment" ``` The rules for creating the API request are the same as when @@ -1306,7 +1306,7 @@ with the exception of `base_sha`, `start_sha`, and `head_sha` attributes. Adds a new note to the thread. ```plaintext -POST /projects/:id/commits/:commit_id/discussions/:discussion_id/notes +POST /projects/:id/repository/commits/:commit_id/discussions/:discussion_id/notes ``` Parameters: @@ -1322,7 +1322,7 @@ Parameters: ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/<discussion_id>/notes?body=comment + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions/<discussion_id>/notes?body=comment ``` ### Modify an existing commit thread note @@ -1330,7 +1330,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>"\ Modify or resolve an existing thread note of a commit. ```plaintext -PUT /projects/:id/commits/:commit_id/discussions/:discussion_id/notes/:note_id +PUT /projects/:id/repository/commits/:commit_id/discussions/:discussion_id/notes/:note_id ``` Parameters: @@ -1345,14 +1345,14 @@ Parameters: ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/<discussion_id>/notes/1108?body=comment" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions/<discussion_id>/notes/1108?body=comment" ``` Resolving a note: ```shell curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/<discussion_id>/notes/1108?resolved=true" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions/<discussion_id>/notes/1108?resolved=true" ``` ### Delete a commit thread note @@ -1360,7 +1360,7 @@ curl --request PUT --header "PRIVATE-TOKEN: <your_access_token>"\ Deletes an existing thread note of a commit. ```plaintext -DELETE /projects/:id/commits/:commit_id/discussions/:discussion_id/notes/:note_id +DELETE /projects/:id/repository/commits/:commit_id/discussions/:discussion_id/notes/:note_id ``` Parameters: @@ -1374,5 +1374,5 @@ Parameters: ```shell curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>"\ - "https://gitlab.example.com/api/v4/projects/5/commits/11/discussions/636" + "https://gitlab.example.com/api/v4/projects/5/repository/commits/11/discussions/636" ``` diff --git a/doc/api/features.md b/doc/api/features.md index d4829f72958..f006fa07188 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -97,16 +97,6 @@ Example response: "type": "development", "group": "group::geo", "default_enabled": true - }, - { - "name": "analytics_devops_adoption_codeowners", - "introduced_by_url": "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59874", - "rollout_issue_url": "https://gitlab.com/gitlab-org/gitlab/-/issues/328542", - "milestone": "13.12", - "log_state_changes": null, - "type": "development", - "group": "group::optimize", - "default_enabled": true } ] ``` diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index fbb583f5a56..b5b920ec0cd 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -495,6 +495,19 @@ Example response: "job_artifacts_synced_in_percentage": "100.00%", "job_artifacts_verified_in_percentage": "100.00%", "job_artifacts_synced_missing_on_primary_count": 0, + "ci_secure_files_count": 5, + "ci_secure_files_checksum_total_count": 5, + "ci_secure_files_checksummed_count": 5, + "ci_secure_files_checksum_failed_count": 0, + "ci_secure_files_synced_count": 5, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 5, + "ci_secure_files_verification_total_count": 5, + "ci_secure_files_verified_count": 5, + "ci_secure_files_verification_failed_count": 0, + "ci_secure_files_synced_in_percentage": "100.00%", + "ci_secure_files_verified_in_percentage": "100.00%", + "ci_secure_files_synced_missing_on_primary_count": 0, }, { "geo_node_id": 2, @@ -830,6 +843,19 @@ Example response: "job_artifacts_synced_in_percentage": "100.00%", "job_artifacts_verified_in_percentage": "100.00%", "job_artifacts_synced_missing_on_primary_count": 0, + "ci_secure_files_count": 5, + "ci_secure_files_checksum_total_count": 5, + "ci_secure_files_checksummed_count": 5, + "ci_secure_files_checksum_failed_count": 0, + "ci_secure_files_synced_count": 5, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 5, + "ci_secure_files_verification_total_count": 5, + "ci_secure_files_verified_count": 5, + "ci_secure_files_verification_failed_count": 0, + "ci_secure_files_synced_in_percentage": "100.00%", + "ci_secure_files_verified_in_percentage": "100.00%", + "ci_secure_files_synced_missing_on_primary_count": 0, } ``` diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index be1bfc79aeb..eab48d65742 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -16,7 +16,7 @@ GraphQL data is arranged in types, so your client can use to consume the API and avoid manual parsing. There are no fixed endpoints and no data model, so you can add -to the API without creating [breaking changes](../../development/contributing/#breaking-changes). +to the API without creating [breaking changes](../../development/deprecation_guidelines/index.md). This enables us to have a [versionless API](https://graphql.org/learn/best-practices/#versioning). ## Vision @@ -69,7 +69,7 @@ However, GitLab sometimes changes the GraphQL API in a way that is not backward- can include removing or renaming fields, arguments, or other parts of the schema. When creating a breaking change, GitLab follows a [deprecation and removal process](#deprecation-and-removal-process). -Learn more about [breaking changes](../../development/contributing/#breaking-changes). +Learn more about [breaking changes](../../development/deprecation_guidelines/index.md). Fields behind a feature flag and disabled by default do not follow the deprecation and removal process, and can be removed at any time without notice. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index fbf6bc116f4..ab5b5a92203 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -91,7 +91,7 @@ four standard [pagination arguments](#connection-pagination-arguments): List of the instance's CI/CD variables. -Returns [`CiVariableConnection`](#civariableconnection). +Returns [`CiInstanceVariableConnection`](#ciinstancevariableconnection). This field returns a [connection](#connections). It accepts the four standard [pagination arguments](#connection-pagination-arguments): @@ -387,7 +387,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="queryrunnersstatus"></a>`status` | [`CiRunnerStatus`](#cirunnerstatus) | Filter runners by status. | | <a id="queryrunnerstaglist"></a>`tagList` | [`[String!]`](#string) | Filter by tags associated with the runner (comma-separated or array). | | <a id="queryrunnerstype"></a>`type` | [`CiRunnerType`](#cirunnertype) | Filter runners by type. | -| <a id="queryrunnersupgradestatus"></a>`upgradeStatus` | [`CiRunnerUpgradeStatusType`](#cirunnerupgradestatustype) | Filter by upgrade status. | +| <a id="queryrunnersupgradestatus"></a>`upgradeStatus` | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | Filter by upgrade status. | ### `Query.snippets` @@ -640,6 +640,8 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationadminsidekiqqueuesdeletejobsartifactsize"></a>`artifactSize` | [`String`](#string) | Delete jobs matching artifact_size in the context metadata. | +| <a id="mutationadminsidekiqqueuesdeletejobsartifactsdependenciescount"></a>`artifactsDependenciesCount` | [`String`](#string) | Delete jobs matching artifacts_dependencies_count in the context metadata. | +| <a id="mutationadminsidekiqqueuesdeletejobsartifactsdependenciessize"></a>`artifactsDependenciesSize` | [`String`](#string) | Delete jobs matching artifacts_dependencies_size in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobscallerid"></a>`callerId` | [`String`](#string) | Delete jobs matching caller_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsclientid"></a>`clientId` | [`String`](#string) | Delete jobs matching client_id in the context metadata. | | <a id="mutationadminsidekiqqueuesdeletejobsclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | @@ -951,6 +953,30 @@ Input type: `BulkEnableDevopsAdoptionNamespacesInput` | <a id="mutationbulkenabledevopsadoptionnamespacesenablednamespaces"></a>`enabledNamespaces` | [`[DevopsAdoptionEnabledNamespace!]`](#devopsadoptionenablednamespace) | Enabled namespaces after mutation. | | <a id="mutationbulkenabledevopsadoptionnamespaceserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.bulkRunnerDelete` + +WARNING: +**Introduced** in 15.3. +This feature is in Alpha. It can be changed or removed at any time. + +Input type: `BulkRunnerDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationbulkrunnerdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationbulkrunnerdeleteids"></a>`ids` | [`[CiRunnerID!]`](#cirunnerid) | IDs of the runners to delete. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationbulkrunnerdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationbulkrunnerdeletedeletedcount"></a>`deletedCount` | [`Int`](#int) | Number of records effectively deleted. Only present if operation was performed synchronously. | +| <a id="mutationbulkrunnerdeletedeletedids"></a>`deletedIds` | [`[CiRunnerID!]`](#cirunnerid) | IDs of records effectively deleted. Only present if operation was performed synchronously. | +| <a id="mutationbulkrunnerdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.ciCdSettingsUpdate` WARNING: @@ -1414,7 +1440,8 @@ Input type: `CreateDiffNoteInput` | ---- | ---- | ----------- | | <a id="mutationcreatediffnotebody"></a>`body` | [`String!`](#string) | Content of the note. | | <a id="mutationcreatediffnoteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationcreatediffnoteconfidential"></a>`confidential` | [`Boolean`](#boolean) | Confidentiality flag of a note. Default is false. | +| <a id="mutationcreatediffnoteconfidential"></a>`confidential` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** This was renamed. Please use `internal`. Deprecated in 15.3. | +| <a id="mutationcreatediffnoteinternal"></a>`internal` | [`Boolean`](#boolean) | Internal flag for a note. Default is false. | | <a id="mutationcreatediffnotenoteableid"></a>`noteableId` | [`NoteableID!`](#noteableid) | Global ID of the resource to add a note to. | | <a id="mutationcreatediffnoteposition"></a>`position` | [`DiffPositionInput!`](#diffpositioninput) | Position of this note on a diff. | @@ -1466,7 +1493,8 @@ Input type: `CreateImageDiffNoteInput` | ---- | ---- | ----------- | | <a id="mutationcreateimagediffnotebody"></a>`body` | [`String!`](#string) | Content of the note. | | <a id="mutationcreateimagediffnoteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationcreateimagediffnoteconfidential"></a>`confidential` | [`Boolean`](#boolean) | Confidentiality flag of a note. Default is false. | +| <a id="mutationcreateimagediffnoteconfidential"></a>`confidential` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** This was renamed. Please use `internal`. Deprecated in 15.3. | +| <a id="mutationcreateimagediffnoteinternal"></a>`internal` | [`Boolean`](#boolean) | Internal flag for a note. Default is false. | | <a id="mutationcreateimagediffnotenoteableid"></a>`noteableId` | [`NoteableID!`](#noteableid) | Global ID of the resource to add a note to. | | <a id="mutationcreateimagediffnoteposition"></a>`position` | [`DiffImagePositionInput!`](#diffimagepositioninput) | Position of this note on a diff. | @@ -1523,7 +1551,7 @@ Input type: `CreateIssueInput` WARNING: **Deprecated** in 14.0. -Manual iteration management is deprecated. Only automatic iteration cadences will be supported in the future. +Use iterationCreate. Input type: `CreateIterationInput` @@ -1535,7 +1563,7 @@ Input type: `CreateIterationInput` | <a id="mutationcreateiterationdescription"></a>`description` | [`String`](#string) | Description of the iteration. | | <a id="mutationcreateiterationduedate"></a>`dueDate` | [`String`](#string) | End date of the iteration. | | <a id="mutationcreateiterationgrouppath"></a>`groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | -| <a id="mutationcreateiterationiterationscadenceid"></a>`iterationsCadenceId` **{warning-solid}** | [`IterationsCadenceID`](#iterationscadenceid) | **Deprecated:** `iterationCadenceId` is deprecated and will be removed in the future. This argument is ignored, because you can't create an iteration in a specific cadence. In the future only automatic iteration cadences will be allowed. Deprecated in 14.10. | +| <a id="mutationcreateiterationiterationscadenceid"></a>`iterationsCadenceId` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iteration cadence to be assigned to the new iteration. | | <a id="mutationcreateiterationprojectpath"></a>`projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | | <a id="mutationcreateiterationstartdate"></a>`startDate` | [`String`](#string) | Start date of the iteration. | | <a id="mutationcreateiterationtitle"></a>`title` | [`String`](#string) | Title of the iteration. | @@ -1563,8 +1591,9 @@ Input type: `CreateNoteInput` | ---- | ---- | ----------- | | <a id="mutationcreatenotebody"></a>`body` | [`String!`](#string) | Content of the note. | | <a id="mutationcreatenoteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationcreatenoteconfidential"></a>`confidential` | [`Boolean`](#boolean) | Confidentiality flag of a note. Default is false. | +| <a id="mutationcreatenoteconfidential"></a>`confidential` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** This was renamed. Please use `internal`. Deprecated in 15.3. | | <a id="mutationcreatenotediscussionid"></a>`discussionId` | [`DiscussionID`](#discussionid) | Global ID of the discussion this note is in reply to. | +| <a id="mutationcreatenoteinternal"></a>`internal` | [`Boolean`](#boolean) | Internal flag for a note. Default is false. | | <a id="mutationcreatenotemergerequestdiffheadsha"></a>`mergeRequestDiffHeadSha` | [`String`](#string) | SHA of the head commit which is used to ensure that the merge request has not been updated since the request was sent. | | <a id="mutationcreatenotenoteableid"></a>`noteableId` | [`NoteableID!`](#noteableid) | Global ID of the resource to add a note to. | @@ -3285,7 +3314,7 @@ Input type: `IterationCadenceCreateInput` | <a id="mutationiterationcadencecreategrouppath"></a>`groupPath` | [`ID!`](#id) | Group where the iteration cadence is created. | | <a id="mutationiterationcadencecreateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="mutationiterationcadencecreaterollover"></a>`rollOver` | [`Boolean`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | -| <a id="mutationiterationcadencecreatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | +| <a id="mutationiterationcadencecreatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the automation start date. | | <a id="mutationiterationcadencecreatetitle"></a>`title` | [`String`](#string) | Title of the iteration cadence. | #### Fields @@ -3331,7 +3360,7 @@ Input type: `IterationCadenceUpdateInput` | <a id="mutationiterationcadenceupdateid"></a>`id` | [`IterationsCadenceID!`](#iterationscadenceid) | Global ID of the iteration cadence. | | <a id="mutationiterationcadenceupdateiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="mutationiterationcadenceupdaterollover"></a>`rollOver` | [`Boolean`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | -| <a id="mutationiterationcadenceupdatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | +| <a id="mutationiterationcadenceupdatestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the automation start date. | | <a id="mutationiterationcadenceupdatetitle"></a>`title` | [`String`](#string) | Title of the iteration cadence. | #### Fields @@ -3344,10 +3373,6 @@ Input type: `IterationCadenceUpdateInput` ### `Mutation.iterationCreate` -WARNING: -**Deprecated** in 14.10. -Manual iteration management is deprecated. Only automatic iteration cadences will be supported in the future. - Input type: `iterationCreateInput` #### Arguments @@ -3358,7 +3383,7 @@ Input type: `iterationCreateInput` | <a id="mutationiterationcreatedescription"></a>`description` | [`String`](#string) | Description of the iteration. | | <a id="mutationiterationcreateduedate"></a>`dueDate` | [`String`](#string) | End date of the iteration. | | <a id="mutationiterationcreategrouppath"></a>`groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | -| <a id="mutationiterationcreateiterationscadenceid"></a>`iterationsCadenceId` **{warning-solid}** | [`IterationsCadenceID`](#iterationscadenceid) | **Deprecated:** `iterationCadenceId` is deprecated and will be removed in the future. This argument is ignored, because you can't create an iteration in a specific cadence. In the future only automatic iteration cadences will be allowed. Deprecated in 14.10. | +| <a id="mutationiterationcreateiterationscadenceid"></a>`iterationsCadenceId` | [`IterationsCadenceID`](#iterationscadenceid) | Global ID of the iteration cadence to be assigned to the new iteration. | | <a id="mutationiterationcreateprojectpath"></a>`projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | | <a id="mutationiterationcreatestartdate"></a>`startDate` | [`String`](#string) | Start date of the iteration. | | <a id="mutationiterationcreatetitle"></a>`title` | [`String`](#string) | Title of the iteration. | @@ -3373,10 +3398,6 @@ Input type: `iterationCreateInput` ### `Mutation.iterationDelete` -WARNING: -**Deprecated** in 14.10. -Manual iteration management is deprecated. Only automatic iteration cadences will be supported in the future. - Input type: `IterationDeleteInput` #### Arguments @@ -3484,6 +3505,7 @@ Input type: `JobRetryInput` | ---- | ---- | ----------- | | <a id="mutationjobretryclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationjobretryid"></a>`id` | [`CiBuildID!`](#cibuildid) | ID of the job to mutate. | +| <a id="mutationjobretryvariables"></a>`variables` | [`[CiVariableInput!]`](#civariableinput) | Variables to use when retrying a manual job. | #### Fields @@ -3608,48 +3630,6 @@ Input type: `MergeRequestCreateInput` | <a id="mutationmergerequestcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationmergerequestcreatemergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | -### `Mutation.mergeRequestRemoveAttentionRequest` - -Input type: `MergeRequestRemoveAttentionRequestInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationmergerequestremoveattentionrequestclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestremoveattentionrequestiid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | -| <a id="mutationmergerequestremoveattentionrequestprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| <a id="mutationmergerequestremoveattentionrequestuserid"></a>`userId` | [`UserID!`](#userid) | User ID of the user for attention request removal. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationmergerequestremoveattentionrequestclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestremoveattentionrequesterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| <a id="mutationmergerequestremoveattentionrequestmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | - -### `Mutation.mergeRequestRequestAttention` - -Input type: `MergeRequestRequestAttentionInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationmergerequestrequestattentionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestrequestattentioniid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | -| <a id="mutationmergerequestrequestattentionprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| <a id="mutationmergerequestrequestattentionuserid"></a>`userId` | [`UserID!`](#userid) | User ID of the user to request attention. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="mutationmergerequestrequestattentionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestrequestattentionerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| <a id="mutationmergerequestrequestattentionmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | - ### `Mutation.mergeRequestReviewerRereview` Input type: `MergeRequestReviewerRereviewInput` @@ -3778,47 +3758,48 @@ Input type: `MergeRequestSetMilestoneInput` | <a id="mutationmergerequestsetmilestoneerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationmergerequestsetmilestonemergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | -### `Mutation.mergeRequestSetSubscription` +### `Mutation.mergeRequestSetReviewers` -Input type: `MergeRequestSetSubscriptionInput` +Input type: `MergeRequestSetReviewersInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="mutationmergerequestsetsubscriptionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestsetsubscriptioniid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | -| <a id="mutationmergerequestsetsubscriptionprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| <a id="mutationmergerequestsetsubscriptionsubscribedstate"></a>`subscribedState` | [`Boolean!`](#boolean) | Desired state of the subscription. | +| <a id="mutationmergerequestsetreviewersclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestsetreviewersiid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | +| <a id="mutationmergerequestsetreviewersoperationmode"></a>`operationMode` | [`MutationOperationMode`](#mutationoperationmode) | Operation to perform. Defaults to REPLACE. | +| <a id="mutationmergerequestsetreviewersprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| <a id="mutationmergerequestsetreviewersreviewerusernames"></a>`reviewerUsernames` | [`[String!]!`](#string) | Usernames of reviewers to assign. Replaces existing reviewers by default. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="mutationmergerequestsetsubscriptionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequestsetsubscriptionerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| <a id="mutationmergerequestsetsubscriptionmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | +| <a id="mutationmergerequestsetreviewersclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestsetreviewerserrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationmergerequestsetreviewersmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | -### `Mutation.mergeRequestToggleAttentionRequested` +### `Mutation.mergeRequestSetSubscription` -Input type: `MergeRequestToggleAttentionRequestedInput` +Input type: `MergeRequestSetSubscriptionInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="mutationmergerequesttoggleattentionrequestedclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequesttoggleattentionrequestediid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | -| <a id="mutationmergerequesttoggleattentionrequestedprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| <a id="mutationmergerequesttoggleattentionrequesteduserid"></a>`userId` | [`UserID!`](#userid) | User ID for the user to toggle attention requested. | +| <a id="mutationmergerequestsetsubscriptionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestsetsubscriptioniid"></a>`iid` | [`String!`](#string) | IID of the merge request to mutate. | +| <a id="mutationmergerequestsetsubscriptionprojectpath"></a>`projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| <a id="mutationmergerequestsetsubscriptionsubscribedstate"></a>`subscribedState` | [`Boolean!`](#boolean) | Desired state of the subscription. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="mutationmergerequesttoggleattentionrequestedclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| <a id="mutationmergerequesttoggleattentionrequestederrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| <a id="mutationmergerequesttoggleattentionrequestedmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | +| <a id="mutationmergerequestsetsubscriptionclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationmergerequestsetsubscriptionerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationmergerequestsetsubscriptionmergerequest"></a>`mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | ### `Mutation.mergeRequestUpdate` @@ -4815,6 +4796,28 @@ Input type: `TimelineEventUpdateInput` | <a id="mutationtimelineeventupdateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationtimelineeventupdatetimelineevent"></a>`timelineEvent` | [`TimelineEventType`](#timelineeventtype) | Timeline event. | +### `Mutation.timelogCreate` + +Input type: `TimelogCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationtimelogcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationtimelogcreateissuableid"></a>`issuableId` | [`IssuableID!`](#issuableid) | Global ID of the issuable (Issue, WorkItem or MergeRequest). | +| <a id="mutationtimelogcreatespentat"></a>`spentAt` | [`Date!`](#date) | When the time was spent. | +| <a id="mutationtimelogcreatesummary"></a>`summary` | [`String!`](#string) | Summary of time spent. | +| <a id="mutationtimelogcreatetimespent"></a>`timeSpent` | [`String!`](#string) | Amount of time spent. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationtimelogcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationtimelogcreateerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationtimelogcreatetimelog"></a>`timelog` | [`Timelog`](#timelog) | Timelog. | + ### `Mutation.timelogDelete` Input type: `TimelogDeleteInput` @@ -4832,7 +4835,7 @@ Input type: `TimelogDeleteInput` | ---- | ---- | ----------- | | <a id="mutationtimelogdeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationtimelogdeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| <a id="mutationtimelogdeletetimelog"></a>`timelog` | [`Timelog`](#timelog) | Deleted timelog. | +| <a id="mutationtimelogdeletetimelog"></a>`timelog` | [`Timelog`](#timelog) | Timelog. | ### `Mutation.todoCreate` @@ -5233,11 +5236,11 @@ Input type: `UpdateIterationInput` | ---- | ---- | ----------- | | <a id="mutationupdateiterationclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | <a id="mutationupdateiterationdescription"></a>`description` | [`String`](#string) | Description of the iteration. | -| <a id="mutationupdateiterationduedate"></a>`dueDate` **{warning-solid}** | [`String`](#string) | **Deprecated:** Manual iteration updates are deprecated, only `description` updates will be allowed in the future. Deprecated in 14.10. | +| <a id="mutationupdateiterationduedate"></a>`dueDate` | [`String`](#string) | End date of the iteration. | | <a id="mutationupdateiterationgrouppath"></a>`groupPath` | [`ID!`](#id) | Group of the iteration. | | <a id="mutationupdateiterationid"></a>`id` | [`ID!`](#id) | Global ID of the iteration. | -| <a id="mutationupdateiterationstartdate"></a>`startDate` **{warning-solid}** | [`String`](#string) | **Deprecated:** Manual iteration updates are deprecated, only `description` updates will be allowed in the future. Deprecated in 14.10. | -| <a id="mutationupdateiterationtitle"></a>`title` **{warning-solid}** | [`String`](#string) | **Deprecated:** Manual iteration updates are deprecated, only `description` updates will be allowed in the future. Deprecated in 14.10. | +| <a id="mutationupdateiterationstartdate"></a>`startDate` | [`String`](#string) | Start date of the iteration. | +| <a id="mutationupdateiterationtitle"></a>`title` | [`String`](#string) | Title of the iteration. | #### Fields @@ -5363,6 +5366,30 @@ Input type: `UpdateSnippetInput` | <a id="mutationupdatesnippeterrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | <a id="mutationupdatesnippetsnippet"></a>`snippet` | [`Snippet`](#snippet) | Snippet after mutation. | +### `Mutation.uploadDelete` + +Deletes an upload. + +Input type: `UploadDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationuploaddeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationuploaddeletefilename"></a>`filename` | [`String!`](#string) | Upload filename. | +| <a id="mutationuploaddeletegrouppath"></a>`groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | +| <a id="mutationuploaddeleteprojectpath"></a>`projectPath` | [`ID`](#id) | Full path of the project with which the resource is associated. | +| <a id="mutationuploaddeletesecret"></a>`secret` | [`String!`](#string) | Secret part of upload path. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="mutationuploaddeleteclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationuploaddeleteerrors"></a>`errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| <a id="mutationuploaddeleteupload"></a>`upload` | [`FileUpload`](#fileupload) | Deleted upload. | + ### `Mutation.userCalloutCreate` Input type: `UserCalloutCreateInput` @@ -5587,6 +5614,7 @@ Input type: `WorkItemCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | | <a id="mutationworkitemcreateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemcreateconfidential"></a>`confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | | <a id="mutationworkitemcreatedescription"></a>`description` | [`String`](#string) | Description of the work item. | | <a id="mutationworkitemcreatehierarchywidget"></a>`hierarchyWidget` | [`WorkItemWidgetHierarchyCreateInput`](#workitemwidgethierarchycreateinput) | Input for hierarchy widget. | | <a id="mutationworkitemcreateprojectpath"></a>`projectPath` | [`ID!`](#id) | Full path of the project the work item is associated with. | @@ -5694,10 +5722,13 @@ Input type: `WorkItemUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="mutationworkitemupdateassigneeswidget"></a>`assigneesWidget` | [`WorkItemWidgetAssigneesInput`](#workitemwidgetassigneesinput) | Input for assignees widget. | | <a id="mutationworkitemupdateclientmutationid"></a>`clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| <a id="mutationworkitemupdateconfidential"></a>`confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | | <a id="mutationworkitemupdatedescriptionwidget"></a>`descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | | <a id="mutationworkitemupdatehierarchywidget"></a>`hierarchyWidget` | [`WorkItemWidgetHierarchyUpdateInput`](#workitemwidgethierarchyupdateinput) | Input for hierarchy widget. | | <a id="mutationworkitemupdateid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| <a id="mutationworkitemupdatestartandduedatewidget"></a>`startAndDueDateWidget` | [`WorkItemWidgetStartAndDueDateUpdateInput`](#workitemwidgetstartandduedateupdateinput) | Input for start and due date widget. | | <a id="mutationworkitemupdatestateevent"></a>`stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | | <a id="mutationworkitemupdatetitle"></a>`title` | [`String`](#string) | Title of the work item. | | <a id="mutationworkitemupdateweightwidget"></a>`weightWidget` | [`WorkItemWidgetWeightInput`](#workitemwidgetweightinput) | Input for weight widget. | @@ -6170,6 +6201,52 @@ The edge type for [`CiGroup`](#cigroup). | <a id="cigroupedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="cigroupedgenode"></a>`node` | [`CiGroup`](#cigroup) | The item at the end of the edge. | +#### `CiGroupVariableConnection` + +The connection type for [`CiGroupVariable`](#cigroupvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cigroupvariableconnectionedges"></a>`edges` | [`[CiGroupVariableEdge]`](#cigroupvariableedge) | A list of edges. | +| <a id="cigroupvariableconnectionnodes"></a>`nodes` | [`[CiGroupVariable]`](#cigroupvariable) | A list of nodes. | +| <a id="cigroupvariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiGroupVariableEdge` + +The edge type for [`CiGroupVariable`](#cigroupvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cigroupvariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="cigroupvariableedgenode"></a>`node` | [`CiGroupVariable`](#cigroupvariable) | The item at the end of the edge. | + +#### `CiInstanceVariableConnection` + +The connection type for [`CiInstanceVariable`](#ciinstancevariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciinstancevariableconnectionedges"></a>`edges` | [`[CiInstanceVariableEdge]`](#ciinstancevariableedge) | A list of edges. | +| <a id="ciinstancevariableconnectionnodes"></a>`nodes` | [`[CiInstanceVariable]`](#ciinstancevariable) | A list of nodes. | +| <a id="ciinstancevariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiInstanceVariableEdge` + +The edge type for [`CiInstanceVariable`](#ciinstancevariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciinstancevariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="ciinstancevariableedgenode"></a>`node` | [`CiInstanceVariable`](#ciinstancevariable) | The item at the end of the edge. | + #### `CiJobArtifactConnection` The connection type for [`CiJobArtifact`](#cijobartifact). @@ -6230,6 +6307,29 @@ The edge type for [`CiJob`](#cijob). | <a id="cijobedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="cijobedgenode"></a>`node` | [`CiJob`](#cijob) | The item at the end of the edge. | +#### `CiManualVariableConnection` + +The connection type for [`CiManualVariable`](#cimanualvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cimanualvariableconnectionedges"></a>`edges` | [`[CiManualVariableEdge]`](#cimanualvariableedge) | A list of edges. | +| <a id="cimanualvariableconnectionnodes"></a>`nodes` | [`[CiManualVariable]`](#cimanualvariable) | A list of nodes. | +| <a id="cimanualvariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiManualVariableEdge` + +The edge type for [`CiManualVariable`](#cimanualvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cimanualvariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="cimanualvariableedgenode"></a>`node` | [`CiManualVariable`](#cimanualvariable) | The item at the end of the edge. | + #### `CiMinutesNamespaceMonthlyUsageConnection` The connection type for [`CiMinutesNamespaceMonthlyUsage`](#ciminutesnamespacemonthlyusage). @@ -6276,6 +6376,29 @@ The edge type for [`CiMinutesProjectMonthlyUsage`](#ciminutesprojectmonthlyusage | <a id="ciminutesprojectmonthlyusageedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="ciminutesprojectmonthlyusageedgenode"></a>`node` | [`CiMinutesProjectMonthlyUsage`](#ciminutesprojectmonthlyusage) | The item at the end of the edge. | +#### `CiProjectVariableConnection` + +The connection type for [`CiProjectVariable`](#ciprojectvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciprojectvariableconnectionedges"></a>`edges` | [`[CiProjectVariableEdge]`](#ciprojectvariableedge) | A list of edges. | +| <a id="ciprojectvariableconnectionnodes"></a>`nodes` | [`[CiProjectVariable]`](#ciprojectvariable) | A list of nodes. | +| <a id="ciprojectvariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiProjectVariableEdge` + +The edge type for [`CiProjectVariable`](#ciprojectvariable). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciprojectvariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="ciprojectvariableedgenode"></a>`node` | [`CiProjectVariable`](#ciprojectvariable) | The item at the end of the edge. | + #### `CiRunnerConnection` The connection type for [`CiRunner`](#cirunner). @@ -6302,51 +6425,51 @@ The edge type for [`CiRunner`](#cirunner). | <a id="cirunneredgenode"></a>`node` | [`CiRunner`](#cirunner) | The item at the end of the edge. | | <a id="cirunneredgeweburl"></a>`webUrl` | [`String`](#string) | Web URL of the runner. The value depends on where you put this field in the query. You can use it for projects or groups. | -#### `CiStageConnection` +#### `CiSecureFileRegistryConnection` -The connection type for [`CiStage`](#cistage). +The connection type for [`CiSecureFileRegistry`](#cisecurefileregistry). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="cistageconnectionedges"></a>`edges` | [`[CiStageEdge]`](#cistageedge) | A list of edges. | -| <a id="cistageconnectionnodes"></a>`nodes` | [`[CiStage]`](#cistage) | A list of nodes. | -| <a id="cistageconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| <a id="cisecurefileregistryconnectionedges"></a>`edges` | [`[CiSecureFileRegistryEdge]`](#cisecurefileregistryedge) | A list of edges. | +| <a id="cisecurefileregistryconnectionnodes"></a>`nodes` | [`[CiSecureFileRegistry]`](#cisecurefileregistry) | A list of nodes. | +| <a id="cisecurefileregistryconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -#### `CiStageEdge` +#### `CiSecureFileRegistryEdge` -The edge type for [`CiStage`](#cistage). +The edge type for [`CiSecureFileRegistry`](#cisecurefileregistry). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="cistageedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | -| <a id="cistageedgenode"></a>`node` | [`CiStage`](#cistage) | The item at the end of the edge. | +| <a id="cisecurefileregistryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="cisecurefileregistryedgenode"></a>`node` | [`CiSecureFileRegistry`](#cisecurefileregistry) | The item at the end of the edge. | -#### `CiVariableConnection` +#### `CiStageConnection` -The connection type for [`CiVariable`](#civariable). +The connection type for [`CiStage`](#cistage). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="civariableconnectionedges"></a>`edges` | [`[CiVariableEdge]`](#civariableedge) | A list of edges. | -| <a id="civariableconnectionnodes"></a>`nodes` | [`[CiVariable]`](#civariable) | A list of nodes. | -| <a id="civariableconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| <a id="cistageconnectionedges"></a>`edges` | [`[CiStageEdge]`](#cistageedge) | A list of edges. | +| <a id="cistageconnectionnodes"></a>`nodes` | [`[CiStage]`](#cistage) | A list of nodes. | +| <a id="cistageconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -#### `CiVariableEdge` +#### `CiStageEdge` -The edge type for [`CiVariable`](#civariable). +The edge type for [`CiStage`](#cistage). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="civariableedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | -| <a id="civariableedgenode"></a>`node` | [`CiVariable`](#civariable) | The item at the end of the edge. | +| <a id="cistageedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="cistageedgenode"></a>`node` | [`CiStage`](#cistage) | The item at the end of the edge. | #### `ClusterAgentActivityEventConnection` @@ -8784,6 +8907,29 @@ The edge type for [`TestSuiteSummary`](#testsuitesummary). | <a id="testsuitesummaryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | | <a id="testsuitesummaryedgenode"></a>`node` | [`TestSuiteSummary`](#testsuitesummary) | The item at the end of the edge. | +#### `TimeTrackingTimelogCategoryConnection` + +The connection type for [`TimeTrackingTimelogCategory`](#timetrackingtimelogcategory). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="timetrackingtimelogcategoryconnectionedges"></a>`edges` | [`[TimeTrackingTimelogCategoryEdge]`](#timetrackingtimelogcategoryedge) | A list of edges. | +| <a id="timetrackingtimelogcategoryconnectionnodes"></a>`nodes` | [`[TimeTrackingTimelogCategory]`](#timetrackingtimelogcategory) | A list of nodes. | +| <a id="timetrackingtimelogcategoryconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `TimeTrackingTimelogCategoryEdge` + +The edge type for [`TimeTrackingTimelogCategory`](#timetrackingtimelogcategory). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="timetrackingtimelogcategoryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. | +| <a id="timetrackingtimelogcategoryedgenode"></a>`node` | [`TimeTrackingTimelogCategory`](#timetrackingtimelogcategory) | The item at the end of the edge. | + #### `TimelineEventTypeConnection` The connection type for [`TimelineEventType`](#timelineeventtype). @@ -9916,6 +10062,40 @@ Represents the total number of issues and their weights for a particular day. | <a id="cigroupname"></a>`name` | [`String`](#string) | Name of the job group. | | <a id="cigroupsize"></a>`size` | [`Int`](#int) | Size of the group. | +### `CiGroupVariable` + +CI/CD variables for a group. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cigroupvariableenvironmentscope"></a>`environmentScope` | [`String`](#string) | Scope defining the environments that can use the variable. | +| <a id="cigroupvariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="cigroupvariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="cigroupvariablemasked"></a>`masked` | [`Boolean`](#boolean) | Indicates whether the variable is masked. | +| <a id="cigroupvariableprotected"></a>`protected` | [`Boolean`](#boolean) | Indicates whether the variable is protected. | +| <a id="cigroupvariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="cigroupvariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="cigroupvariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + +### `CiInstanceVariable` + +CI/CD variables for a GitLab instance. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciinstancevariableenvironmentscope"></a>`environmentScope` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.3. No longer used, only available for GroupVariableType and ProjectVariableType. | +| <a id="ciinstancevariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="ciinstancevariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="ciinstancevariablemasked"></a>`masked` | [`Boolean`](#boolean) | Indicates whether the variable is masked. | +| <a id="ciinstancevariableprotected"></a>`protected` | [`Boolean`](#boolean) | Indicates whether the variable is protected. | +| <a id="ciinstancevariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="ciinstancevariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="ciinstancevariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + ### `CiJob` #### Fields @@ -9937,7 +10117,7 @@ Represents the total number of issues and their weights for a particular day. | <a id="cijobid"></a>`id` | [`JobID`](#jobid) | ID of the job. | | <a id="cijobkind"></a>`kind` | [`CiJobKind!`](#cijobkind) | Indicates the type of job. | | <a id="cijobmanualjob"></a>`manualJob` | [`Boolean`](#boolean) | Whether the job has a manual action. | -| <a id="cijobmanualvariables"></a>`manualVariables` | [`CiVariableConnection`](#civariableconnection) | Variables added to a manual job when the job is triggered. (see [Connections](#connections)) | +| <a id="cijobmanualvariables"></a>`manualVariables` | [`CiManualVariableConnection`](#cimanualvariableconnection) | Variables added to a manual job when the job is triggered. (see [Connections](#connections)) | | <a id="cijobname"></a>`name` | [`String`](#string) | Name of the job. | | <a id="cijobneeds"></a>`needs` | [`CiBuildNeedConnection`](#cibuildneedconnection) | References to builds that must complete before the jobs run. (see [Connections](#connections)) | | <a id="cijobpipeline"></a>`pipeline` | [`Pipeline`](#pipeline) | Pipeline the job belongs to. | @@ -9978,6 +10158,21 @@ Represents the total number of issues and their weights for a particular day. | ---- | ---- | ----------- | | <a id="cijobtokenscopetypeprojects"></a>`projects` | [`ProjectConnection!`](#projectconnection) | Allow list of projects that can be accessed by CI Job tokens created by this project. (see [Connections](#connections)) | +### `CiManualVariable` + +CI/CD variables given to a manual job. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cimanualvariableenvironmentscope"></a>`environmentScope` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.3. No longer used, only available for GroupVariableType and ProjectVariableType. | +| <a id="cimanualvariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="cimanualvariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="cimanualvariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="cimanualvariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="cimanualvariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + ### `CiMinutesNamespaceMonthlyUsage` #### Fields @@ -10000,6 +10195,23 @@ Represents the total number of issues and their weights for a particular day. | <a id="ciminutesprojectmonthlyusagename"></a>`name` | [`String`](#string) | Name of the project. | | <a id="ciminutesprojectmonthlyusagesharedrunnersduration"></a>`sharedRunnersDuration` | [`Int`](#int) | Total duration (in seconds) of shared runners use by the project for the month. | +### `CiProjectVariable` + +CI/CD variables for a project. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="ciprojectvariableenvironmentscope"></a>`environmentScope` | [`String`](#string) | Scope defining the environments that can use the variable. | +| <a id="ciprojectvariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="ciprojectvariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="ciprojectvariablemasked"></a>`masked` | [`Boolean`](#boolean) | Indicates whether the variable is masked. | +| <a id="ciprojectvariableprotected"></a>`protected` | [`Boolean`](#boolean) | Indicates whether the variable is protected. | +| <a id="ciprojectvariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="ciprojectvariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="ciprojectvariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + ### `CiRunner` #### Fields @@ -10036,7 +10248,7 @@ Represents the total number of issues and their weights for a particular day. | <a id="cirunnershortsha"></a>`shortSha` | [`String`](#string) | First eight characters of the runner's token used to authenticate new job requests. Used as the runner's unique ID. | | <a id="cirunnertaglist"></a>`tagList` | [`[String!]`](#string) | Tags associated with the runner. | | <a id="cirunnertokenexpiresat"></a>`tokenExpiresAt` | [`Time`](#time) | Runner token expiration time. | -| <a id="cirunnerupgradestatus"></a>`upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatusType`](#cirunnerupgradestatustype) | **Introduced** in 14.10. This feature is in Alpha. It can be changed or removed at any time. | +| <a id="cirunnerupgradestatus"></a>`upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | **Introduced** in 14.10. This feature is in Alpha. It can be changed or removed at any time. Availability of upgrades for the runner. | | <a id="cirunneruserpermissions"></a>`userPermissions` | [`RunnerPermissions!`](#runnerpermissions) | Permissions for the current user on the resource. | | <a id="cirunnerversion"></a>`version` | [`String`](#string) | Version of the runner. | @@ -10070,6 +10282,25 @@ Returns [`CiRunnerStatus!`](#cirunnerstatus). | ---- | ---- | ----------- | | <a id="cirunnerstatuslegacymode"></a>`legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. In GitLab 16.0 and later, the field will act as if `legacyMode` is null. | +### `CiSecureFileRegistry` + +Represents the Geo replication and verification state of a ci_secure_file. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="cisecurefileregistrycisecurefileid"></a>`ciSecureFileId` | [`ID!`](#id) | ID of the Ci Secure File. | +| <a id="cisecurefileregistrycreatedat"></a>`createdAt` | [`Time`](#time) | Timestamp when the CiSecureFileRegistry was created. | +| <a id="cisecurefileregistryid"></a>`id` | [`ID!`](#id) | ID of the CiSecureFileRegistry. | +| <a id="cisecurefileregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the CiSecureFileRegistry. | +| <a id="cisecurefileregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the CiSecureFileRegistry. | +| <a id="cisecurefileregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is resynced. | +| <a id="cisecurefileregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the CiSecureFileRegistry. | +| <a id="cisecurefileregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the CiSecureFileRegistry. | +| <a id="cisecurefileregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is reverified. | +| <a id="cisecurefileregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the CiSecureFileRegistry. | + ### `CiStage` #### Fields @@ -10094,21 +10325,6 @@ GitLab CI/CD configuration template. | <a id="citemplatecontent"></a>`content` | [`String!`](#string) | Contents of the CI template. | | <a id="citemplatename"></a>`name` | [`String!`](#string) | Name of the CI template. | -### `CiVariable` - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| <a id="civariableenvironmentscope"></a>`environmentScope` | [`String`](#string) | Scope defining the environments in which the variable can be used. | -| <a id="civariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | -| <a id="civariablekey"></a>`key` | [`String`](#string) | Name of the variable. | -| <a id="civariablemasked"></a>`masked` | [`Boolean`](#boolean) | Indicates whether the variable is masked. | -| <a id="civariableprotected"></a>`protected` | [`Boolean`](#boolean) | Indicates whether the variable is protected. | -| <a id="civariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | -| <a id="civariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | -| <a id="civariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | - ### `ClusterAgent` #### Fields @@ -10342,6 +10558,18 @@ Connection details for an Agent. | <a id="connectedagentconnectionid"></a>`connectionId` | [`BigInt`](#bigint) | ID of the connection. | | <a id="connectedagentmetadata"></a>`metadata` | [`AgentMetadata`](#agentmetadata) | Information about the Agent. | +### `ContactStateCounts` + +Represents the total number of contacts for the represented states. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="contactstatecountsactive"></a>`active` | [`Int`](#int) | Number of contacts with state `ACTIVE`. | +| <a id="contactstatecountsall"></a>`all` | [`Int`](#int) | Number of contacts with state `ALL`. | +| <a id="contactstatecountsinactive"></a>`inactive` | [`Int`](#int) | Number of contacts with state `INACTIVE`. | + ### `ContainerExpirationPolicy` A tag expiration policy designed to keep only the images that matter most. @@ -11123,7 +11351,7 @@ Returns [`[DoraMetric!]`](#dorametric). | ---- | ---- | ----------- | | <a id="dorametricsenddate"></a>`endDate` | [`Date`](#date) | Date range to end at. Default is the current date. | | <a id="dorametricsenvironmenttier"></a>`environmentTier` | [`DeploymentTier`](#deploymenttier) | Deployment tier of the environments to return. Deprecated, please update to `environment_tiers` param. | -| <a id="dorametricsenvironmenttiers"></a>`environmentTiers` | [`[DeploymentTier!]`](#deploymenttier) | Deployment tiers of the environments to return. Defaults to [`PRODUCTION`]. | +| <a id="dorametricsenvironmenttiers"></a>`environmentTiers` | [`[DeploymentTier!]`](#deploymenttier) | Deployment tiers of the environments to return. Defaults to `[PRODUCTION]`. | | <a id="dorametricsinterval"></a>`interval` | [`DoraMetricBucketingInterval`](#dorametricbucketinginterval) | How the metric should be aggregrated. Defaults to `DAILY`. In the case of `ALL`, the `date` field in the response will be `null`. | | <a id="dorametricsmetric"></a>`metric` | [`DoraMetricType!`](#dorametrictype) | Type of metric to return. | | <a id="dorametricsstartdate"></a>`startDate` | [`Date`](#date) | Date range to start from. Default is 3 months ago. | @@ -11567,7 +11795,7 @@ Represents epic board list metadata. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="epiclistmetadataepicscount"></a>`epicsCount` | [`Int`](#int) | Count of epics in the list. | -| <a id="epiclistmetadatatotalweight"></a>`totalWeight` | [`Int`](#int) | Total weight of all issues in the list. Available only when feature flag `epic_board_total_weight` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. | +| <a id="epiclistmetadatatotalweight"></a>`totalWeight` **{warning-solid}** | [`Int`](#int) | **Introduced** in 14.7. This feature is in Alpha. It can be changed or removed at any time. Total weight of all issues in the list. | ### `EpicPermissions` @@ -11637,7 +11865,7 @@ Represents an external resource to send audit events to. | ---- | ---- | ----------- | | <a id="externalauditeventdestinationdestinationurl"></a>`destinationUrl` | [`String!`](#string) | External destination to send audit events to. | | <a id="externalauditeventdestinationgroup"></a>`group` | [`Group!`](#group) | Group the destination belongs to. | -| <a id="externalauditeventdestinationheaders"></a>`headers` | [`AuditEventStreamingHeaderConnection!`](#auditeventstreamingheaderconnection) | List of additional HTTP headers sent with each event. Available only when feature flag `streaming_audit_event_headers` is enabled. This flag is enabled by default. (see [Connections](#connections)) | +| <a id="externalauditeventdestinationheaders"></a>`headers` | [`AuditEventStreamingHeaderConnection!`](#auditeventstreamingheaderconnection) | List of additional HTTP headers sent with each event. (see [Connections](#connections)) | | <a id="externalauditeventdestinationid"></a>`id` | [`ID!`](#id) | ID of the destination. | | <a id="externalauditeventdestinationverificationtoken"></a>`verificationToken` | [`String!`](#string) | Verification token to validate source of event. | @@ -11657,6 +11885,16 @@ Represents an external issue. | <a id="externalissueupdatedat"></a>`updatedAt` | [`Time`](#time) | Timestamp of when the issue was updated. | | <a id="externalissueweburl"></a>`webUrl` | [`String`](#string) | URL to the issue in the external tracker. | +### `FileUpload` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="fileuploadid"></a>`id` | [`UploadID!`](#uploadid) | Global ID of the upload. | +| <a id="fileuploadpath"></a>`path` | [`String!`](#string) | Path of the upload. | +| <a id="fileuploadsize"></a>`size` | [`Int!`](#int) | Size of the upload in bytes. | + ### `GeoNode` #### Fields @@ -11681,6 +11919,24 @@ Represents an external issue. #### Fields with arguments +##### `GeoNode.ciSecureFileRegistries` + +Find Ci Secure File registries on this Geo node. + +Returns [`CiSecureFileRegistryConnection`](#cisecurefileregistryconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="geonodecisecurefileregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodecisecurefileregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodecisecurefileregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | + ##### `GeoNode.groupWikiRepositoryRegistries` Find group wiki repository registries on this Geo node. @@ -11696,6 +11952,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodegroupwikirepositoryregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodegroupwikirepositoryregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodegroupwikirepositoryregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.jobArtifactRegistries` @@ -11712,6 +11970,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodejobartifactregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodejobartifactregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodejobartifactregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.lfsObjectRegistries` @@ -11728,6 +11988,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodelfsobjectregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodelfsobjectregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodelfsobjectregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.mergeRequestDiffRegistries` @@ -11744,6 +12006,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodemergerequestdiffregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodemergerequestdiffregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodemergerequestdiffregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.packageFileRegistries` @@ -11760,6 +12024,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodepackagefileregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodepackagefileregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodepackagefileregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.pagesDeploymentRegistries` @@ -11776,6 +12042,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodepagesdeploymentregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodepagesdeploymentregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodepagesdeploymentregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.pipelineArtifactRegistries` @@ -11792,6 +12060,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodepipelineartifactregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodepipelineartifactregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodepipelineartifactregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.snippetRepositoryRegistries` @@ -11808,6 +12078,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodesnippetrepositoryregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodesnippetrepositoryregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodesnippetrepositoryregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.terraformStateVersionRegistries` @@ -11824,6 +12096,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodeterraformstateversionregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodeterraformstateversionregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodeterraformstateversionregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.uploadRegistries` @@ -11840,6 +12114,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | <a id="geonodeuploadregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| <a id="geonodeuploadregistriesreplicationstate"></a>`replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| <a id="geonodeuploadregistriesverificationstate"></a>`verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ### `GrafanaIntegration` @@ -11864,7 +12140,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupallowstalerunnerpruning"></a>`allowStaleRunnerPruning` | [`Boolean!`](#boolean) | Indicates whether to regularly prune stale group runners. Defaults to false. | | <a id="groupautodevopsenabled"></a>`autoDevopsEnabled` | [`Boolean`](#boolean) | Indicates whether Auto DevOps is enabled for all projects within this group. | | <a id="groupavatarurl"></a>`avatarUrl` | [`String`](#string) | Avatar URL of the group. | -| <a id="groupcivariables"></a>`ciVariables` | [`CiVariableConnection`](#civariableconnection) | List of the group's CI/CD variables. (see [Connections](#connections)) | +| <a id="groupcivariables"></a>`ciVariables` | [`CiGroupVariableConnection`](#cigroupvariableconnection) | List of the group's CI/CD variables. (see [Connections](#connections)) | | <a id="groupcontainerrepositoriescount"></a>`containerRepositoriesCount` | [`Int!`](#int) | Number of container repositories in the group. | | <a id="groupcontainslockedprojects"></a>`containsLockedProjects` | [`Boolean!`](#boolean) | Includes at least one project where the repository size exceeds the limit. | | <a id="groupcrossprojectpipelineavailable"></a>`crossProjectPipelineAvailable` | [`Boolean!`](#boolean) | Indicates if the cross_project_pipeline feature is available for the namespace. | @@ -11907,6 +12183,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="groupstoragesizelimit"></a>`storageSizeLimit` | [`Float`](#float) | Total storage limit of the root namespace in bytes. | | <a id="groupsubgroupcreationlevel"></a>`subgroupCreationLevel` | [`String`](#string) | Permission level required to create subgroups within the group. | | <a id="grouptemporarystorageincreaseendson"></a>`temporaryStorageIncreaseEndsOn` | [`Time`](#time) | Date until the temporary storage increase is active. | +| <a id="grouptimelogcategories"></a>`timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the namespace. | | <a id="grouptotalrepositorysize"></a>`totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | <a id="grouptotalrepositorysizeexcess"></a>`totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. | | <a id="grouptwofactorgraceperiod"></a>`twoFactorGracePeriod` | [`Int`](#int) | Time before two-factor authentication is enforced. | @@ -12005,6 +12282,19 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="groupcomplianceframeworksid"></a>`id` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | Global ID of a specific compliance framework to return. | +##### `Group.contactStateCounts` + +Counts of contacts by state for the group. + +Returns [`ContactStateCounts`](#contactstatecounts). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="groupcontactstatecountssearch"></a>`search` | [`String`](#string) | Search term to find contacts with. | +| <a id="groupcontactstatecountsstate"></a>`state` | [`CustomerRelationsContactState`](#customerrelationscontactstate) | State of the contacts to search for. | + ##### `Group.contacts` Find contacts of this group. @@ -12021,6 +12311,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | <a id="groupcontactsids"></a>`ids` | [`[CustomerRelationsContactID!]`](#customerrelationscontactid) | Filter contacts by IDs. | | <a id="groupcontactssearch"></a>`search` | [`String`](#string) | Search term to find contacts with. | +| <a id="groupcontactssort"></a>`sort` | [`ContactSort`](#contactsort) | Criteria to sort contacts by. | | <a id="groupcontactsstate"></a>`state` | [`CustomerRelationsContactState`](#customerrelationscontactstate) | State of the contacts to search for. | ##### `Group.containerRepositories` @@ -12443,7 +12734,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="grouprunnersstatus"></a>`status` | [`CiRunnerStatus`](#cirunnerstatus) | Filter runners by status. | | <a id="grouprunnerstaglist"></a>`tagList` | [`[String!]`](#string) | Filter by tags associated with the runner (comma-separated or array). | | <a id="grouprunnerstype"></a>`type` | [`CiRunnerType`](#cirunnertype) | Filter runners by type. | -| <a id="grouprunnersupgradestatus"></a>`upgradeStatus` | [`CiRunnerUpgradeStatusType`](#cirunnerupgradestatustype) | Filter by upgrade status. | +| <a id="grouprunnersupgradestatus"></a>`upgradeStatus` | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | Filter by upgrade status. | ##### `Group.scanExecutionPolicies` @@ -12662,9 +12953,11 @@ Represents the Geo sync and verification state of a group wiki repository. | <a id="groupwikirepositoryregistryid"></a>`id` | [`ID!`](#id) | ID of the GroupWikiRepositoryRegistry. | | <a id="groupwikirepositoryregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the GroupWikiRepositoryRegistry. | | <a id="groupwikirepositoryregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the GroupWikiRepositoryRegistry. | -| <a id="groupwikirepositoryregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the GroupWikiRepositoryRegistry should be resynced. | +| <a id="groupwikirepositoryregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the GroupWikiRepositoryRegistry is resynced. | | <a id="groupwikirepositoryregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the GroupWikiRepositoryRegistry. | | <a id="groupwikirepositoryregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the GroupWikiRepositoryRegistry. | +| <a id="groupwikirepositoryregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the GroupWikiRepositoryRegistry is reverified. | +| <a id="groupwikirepositoryregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the GroupWikiRepositoryRegistry. | ### `HelmFileMetadata` @@ -13030,7 +13323,7 @@ Represents an iteration cadence. | <a id="iterationcadenceid"></a>`id` | [`IterationsCadenceID!`](#iterationscadenceid) | Global ID of the iteration cadence. | | <a id="iterationcadenceiterationsinadvance"></a>`iterationsInAdvance` | [`Int`](#int) | Upcoming iterations to be created when iteration cadence is set to automatic. | | <a id="iterationcadencerollover"></a>`rollOver` | [`Boolean!`](#boolean) | Whether the iteration cadence should roll over issues to the next iteration or not. | -| <a id="iterationcadencestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the iteration cadence start date. | +| <a id="iterationcadencestartdate"></a>`startDate` | [`Time`](#time) | Timestamp of the automation start date. | | <a id="iterationcadencetitle"></a>`title` | [`String!`](#string) | Title of the iteration cadence. | ### `JiraImport` @@ -13111,9 +13404,11 @@ Represents the Geo replication and verification state of a job_artifact. | <a id="jobartifactregistryid"></a>`id` | [`ID!`](#id) | ID of the JobArtifactRegistry. | | <a id="jobartifactregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the JobArtifactRegistry. | | <a id="jobartifactregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the JobArtifactRegistry. | -| <a id="jobartifactregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the JobArtifactRegistry should be resynced. | +| <a id="jobartifactregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the JobArtifactRegistry is resynced. | | <a id="jobartifactregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the JobArtifactRegistry. | | <a id="jobartifactregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the JobArtifactRegistry. | +| <a id="jobartifactregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the JobArtifactRegistry is reverified. | +| <a id="jobartifactregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the JobArtifactRegistry. | ### `JobPermissions` @@ -13163,9 +13458,11 @@ Represents the Geo sync and verification state of an LFS object. | <a id="lfsobjectregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the LfsObjectRegistry. | | <a id="lfsobjectregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the LfsObjectRegistry. | | <a id="lfsobjectregistrylfsobjectid"></a>`lfsObjectId` | [`ID!`](#id) | ID of the LFS object. | -| <a id="lfsobjectregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the LfsObjectRegistry should be resynced. | +| <a id="lfsobjectregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the LfsObjectRegistry is resynced. | | <a id="lfsobjectregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the LfsObjectRegistry. | | <a id="lfsobjectregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the LfsObjectRegistry. | +| <a id="lfsobjectregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the LfsObjectRegistry is reverified. | +| <a id="lfsobjectregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the LfsObjectRegistry. | ### `LicenseHistoryEntry` @@ -13231,6 +13528,7 @@ Maven metadata. | <a id="mergerequestdefaultsquashcommitmessage"></a>`defaultSquashCommitMessage` | [`String`](#string) | Default squash commit message of the merge request. | | <a id="mergerequestdescription"></a>`description` | [`String`](#string) | Description of the merge request (Markdown rendered as HTML for caching). | | <a id="mergerequestdescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | +| <a id="mergerequestdetailedmergestatus"></a>`detailedMergeStatus` **{warning-solid}** | [`DetailedMergeStatus`](#detailedmergestatus) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Detailed merge status of the merge request. | | <a id="mergerequestdiffheadsha"></a>`diffHeadSha` | [`String`](#string) | Diff head SHA of the merge request. | | <a id="mergerequestdiffrefs"></a>`diffRefs` | [`DiffRefs`](#diffrefs) | References of the base SHA, the head SHA, and the start SHA for this merge request. | | <a id="mergerequestdiffstatssummary"></a>`diffStatsSummary` | [`DiffStatsSummary`](#diffstatssummary) | Summary of which files were changed in this merge request. | @@ -13847,9 +14145,11 @@ Represents the Geo sync and verification state of a Merge Request diff. | <a id="mergerequestdiffregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the MergeRequestDiffRegistry. | | <a id="mergerequestdiffregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the MergeRequestDiffRegistry. | | <a id="mergerequestdiffregistrymergerequestdiffid"></a>`mergeRequestDiffId` | [`ID!`](#id) | ID of the Merge Request diff. | -| <a id="mergerequestdiffregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the MergeRequestDiffRegistry should be resynced. | +| <a id="mergerequestdiffregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the MergeRequestDiffRegistry is resynced. | | <a id="mergerequestdiffregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the MergeRequestDiffRegistry. | | <a id="mergerequestdiffregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the MergeRequestDiffRegistry. | +| <a id="mergerequestdiffregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the MergeRequestDiffRegistry is reverified. | +| <a id="mergerequestdiffregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the MergeRequestDiffRegistry. | ### `MergeRequestParticipant` @@ -14469,6 +14769,7 @@ Contains statistics about a milestone. | <a id="namespacesharedrunnerssetting"></a>`sharedRunnersSetting` | [`SharedRunnersSetting`](#sharedrunnerssetting) | Shared runners availability for the namespace and its descendants. | | <a id="namespacestoragesizelimit"></a>`storageSizeLimit` | [`Float`](#float) | Total storage limit of the root namespace in bytes. | | <a id="namespacetemporarystorageincreaseendson"></a>`temporaryStorageIncreaseEndsOn` | [`Time`](#time) | Date until the temporary storage increase is active. | +| <a id="namespacetimelogcategories"></a>`timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the namespace. | | <a id="namespacetotalrepositorysize"></a>`totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | <a id="namespacetotalrepositorysizeexcess"></a>`totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. | | <a id="namespacevisibility"></a>`visibility` | [`String`](#string) | Visibility of the namespace. | @@ -14574,10 +14875,11 @@ Represents the network policy. | <a id="noteauthor"></a>`author` | [`UserCore!`](#usercore) | User who wrote this note. | | <a id="notebody"></a>`body` | [`String!`](#string) | Content of the note. | | <a id="notebodyhtml"></a>`bodyHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `note`. | -| <a id="noteconfidential"></a>`confidential` | [`Boolean`](#boolean) | Indicates if this note is confidential. | +| <a id="noteconfidential"></a>`confidential` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.3. This was renamed. Use: `internal`. | | <a id="notecreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp of the note creation. | | <a id="notediscussion"></a>`discussion` | [`Discussion`](#discussion) | Discussion this note is a part of. | | <a id="noteid"></a>`id` | [`NoteID!`](#noteid) | ID of the note. | +| <a id="noteinternal"></a>`internal` | [`Boolean`](#boolean) | Indicates if this note is internal. | | <a id="noteposition"></a>`position` | [`DiffPosition`](#diffposition) | Position of this note on a diff. | | <a id="noteproject"></a>`project` | [`Project`](#project) | Project associated with the note. | | <a id="noteresolvable"></a>`resolvable` | [`Boolean!`](#boolean) | Indicates if the object can be resolved. | @@ -14794,9 +15096,11 @@ Represents the Geo sync and verification state of a package file. | <a id="packagefileregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the PackageFileRegistry. | | <a id="packagefileregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PackageFileRegistry. | | <a id="packagefileregistrypackagefileid"></a>`packageFileId` | [`ID!`](#id) | ID of the PackageFile. | -| <a id="packagefileregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PackageFileRegistry should be resynced. | +| <a id="packagefileregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PackageFileRegistry is resynced. | | <a id="packagefileregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PackageFileRegistry. | | <a id="packagefileregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the PackageFileRegistry. | +| <a id="packagefileregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the PackageFileRegistry is reverified. | +| <a id="packagefileregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the PackageFileRegistry. | ### `PackageHelmDependencyType` @@ -14916,9 +15220,11 @@ Represents the Geo replication and verification state of a pages_deployment. | <a id="pagesdeploymentregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the PagesDeploymentRegistry. | | <a id="pagesdeploymentregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PagesDeploymentRegistry. | | <a id="pagesdeploymentregistrypagesdeploymentid"></a>`pagesDeploymentId` | [`ID!`](#id) | ID of the Pages Deployment. | -| <a id="pagesdeploymentregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PagesDeploymentRegistry should be resynced. | +| <a id="pagesdeploymentregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PagesDeploymentRegistry is resynced. | | <a id="pagesdeploymentregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PagesDeploymentRegistry. | | <a id="pagesdeploymentregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the PagesDeploymentRegistry. | +| <a id="pagesdeploymentregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the PagesDeploymentRegistry is reverified. | +| <a id="pagesdeploymentregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the PagesDeploymentRegistry. | ### `PathLock` @@ -15085,9 +15391,11 @@ Represents the Geo sync and verification state of a pipeline artifact. | <a id="pipelineartifactregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the PipelineArtifactRegistry. | | <a id="pipelineartifactregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PipelineArtifactRegistry. | | <a id="pipelineartifactregistrypipelineartifactid"></a>`pipelineArtifactId` | [`ID!`](#id) | ID of the pipeline artifact. | -| <a id="pipelineartifactregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PipelineArtifactRegistry should be resynced. | +| <a id="pipelineartifactregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PipelineArtifactRegistry is resynced. | | <a id="pipelineartifactregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PipelineArtifactRegistry. | | <a id="pipelineartifactregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the PipelineArtifactRegistry. | +| <a id="pipelineartifactregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the PipelineArtifactRegistry is reverified. | +| <a id="pipelineartifactregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the PipelineArtifactRegistry. | ### `PipelineCounts` @@ -15164,7 +15472,7 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="projectcicdsettings"></a>`ciCdSettings` | [`ProjectCiCdSetting`](#projectcicdsetting) | CI/CD settings for the project. | | <a id="projectciconfigpathordefault"></a>`ciConfigPathOrDefault` | [`String!`](#string) | Path of the CI configuration file. | | <a id="projectcijobtokenscope"></a>`ciJobTokenScope` | [`CiJobTokenScopeType`](#cijobtokenscopetype) | The CI Job Tokens scope of access. | -| <a id="projectcivariables"></a>`ciVariables` | [`CiVariableConnection`](#civariableconnection) | List of the project's CI/CD variables. (see [Connections](#connections)) | +| <a id="projectcivariables"></a>`ciVariables` | [`CiProjectVariableConnection`](#ciprojectvariableconnection) | List of the project's CI/CD variables. (see [Connections](#connections)) | | <a id="projectcodecoveragesummary"></a>`codeCoverageSummary` | [`CodeCoverageSummary`](#codecoveragesummary) | Code coverage summary associated with the project. | | <a id="projectcomplianceframeworks"></a>`complianceFrameworks` | [`ComplianceFrameworkConnection`](#complianceframeworkconnection) | Compliance frameworks associated with the project. (see [Connections](#connections)) | | <a id="projectcontainerexpirationpolicy"></a>`containerExpirationPolicy` | [`ContainerExpirationPolicy`](#containerexpirationpolicy) | Container expiration policy of the project. | @@ -15229,6 +15537,7 @@ Represents vulnerability finding of a security report on the pipeline. | <a id="projectsuggestioncommitmessage"></a>`suggestionCommitMessage` | [`String`](#string) | Commit message used to apply merge request suggestions. | | <a id="projecttaglist"></a>`tagList` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.12. Use `topics`. | | <a id="projectterraformstates"></a>`terraformStates` | [`TerraformStateConnection`](#terraformstateconnection) | Terraform states associated with the project. (see [Connections](#connections)) | +| <a id="projecttimelogcategories"></a>`timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the project. | | <a id="projecttopics"></a>`topics` | [`[String!]`](#string) | List of project topics. | | <a id="projectuserpermissions"></a>`userPermissions` | [`ProjectPermissions!`](#projectpermissions) | Permissions for the current user on the resource. | | <a id="projectvisibility"></a>`visibility` | [`String`](#string) | Visibility of the project. | @@ -15510,6 +15819,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="projectenvironmentssearch"></a>`search` | [`String`](#string) | Search query for environment name. | | <a id="projectenvironmentsstates"></a>`states` | [`[String!]`](#string) | States of environments that should be included in result. | +##### `Project.forkTargets` + +Namespaces in which the current user can fork the project into. + +Returns [`NamespaceConnection`](#namespaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="projectforktargetssearch"></a>`search` | [`String`](#string) | Search query for path or name. | + ##### `Project.incidentManagementEscalationPolicies` Incident Management escalation policies of the project. @@ -17252,10 +17577,12 @@ Represents the Geo sync and verification state of a snippet repository. | <a id="snippetrepositoryregistryid"></a>`id` | [`ID!`](#id) | ID of the SnippetRepositoryRegistry. | | <a id="snippetrepositoryregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the SnippetRepositoryRegistry. | | <a id="snippetrepositoryregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the SnippetRepositoryRegistry. | -| <a id="snippetrepositoryregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the SnippetRepositoryRegistry should be resynced. | +| <a id="snippetrepositoryregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the SnippetRepositoryRegistry is resynced. | | <a id="snippetrepositoryregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the SnippetRepositoryRegistry. | | <a id="snippetrepositoryregistrysnippetrepositoryid"></a>`snippetRepositoryId` | [`ID!`](#id) | ID of the Snippet Repository. | | <a id="snippetrepositoryregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the SnippetRepositoryRegistry. | +| <a id="snippetrepositoryregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the SnippetRepositoryRegistry is reverified. | +| <a id="snippetrepositoryregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the SnippetRepositoryRegistry. | ### `StatusAction` @@ -17354,10 +17681,12 @@ Represents the Geo sync and verification state of a terraform state version. | <a id="terraformstateversionregistryid"></a>`id` | [`ID!`](#id) | ID of the TerraformStateVersionRegistry. | | <a id="terraformstateversionregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the TerraformStateVersionRegistry. | | <a id="terraformstateversionregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the TerraformStateVersionRegistry. | -| <a id="terraformstateversionregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the TerraformStateVersionRegistry should be resynced. | +| <a id="terraformstateversionregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the TerraformStateVersionRegistry is resynced. | | <a id="terraformstateversionregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the TerraformStateVersionRegistry. | | <a id="terraformstateversionregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the TerraformStateVersionRegistry. | | <a id="terraformstateversionregistryterraformstateversionid"></a>`terraformStateVersionId` | [`ID!`](#id) | ID of the terraform state version. | +| <a id="terraformstateversionregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the TerraformStateVersionRegistry is reverified. | +| <a id="terraformstateversionregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the TerraformStateVersionRegistry. | ### `TestCase` @@ -17465,6 +17794,21 @@ Represents the time report stats for timeboxes. | <a id="timereportstatsincomplete"></a>`incomplete` | [`TimeboxMetrics`](#timeboxmetrics) | Incomplete issues metrics. | | <a id="timereportstatstotal"></a>`total` | [`TimeboxMetrics`](#timeboxmetrics) | Total issues metrics. | +### `TimeTrackingTimelogCategory` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="timetrackingtimelogcategorybillable"></a>`billable` | [`Boolean`](#boolean) | Whether the category is billable or not. | +| <a id="timetrackingtimelogcategorybillingrate"></a>`billingRate` | [`Float`](#float) | Billing rate for the category. | +| <a id="timetrackingtimelogcategorycolor"></a>`color` | [`Color`](#color) | Color assigned to the category. | +| <a id="timetrackingtimelogcategorycreatedat"></a>`createdAt` | [`Time!`](#time) | When the category was created. | +| <a id="timetrackingtimelogcategorydescription"></a>`description` | [`String`](#string) | Description of the category. | +| <a id="timetrackingtimelogcategoryid"></a>`id` | [`ID!`](#id) | Internal ID of the timelog category. | +| <a id="timetrackingtimelogcategoryname"></a>`name` | [`String!`](#string) | Name of the category. | +| <a id="timetrackingtimelogcategoryupdatedat"></a>`updatedAt` | [`Time!`](#time) | When the category was last updated. | + ### `TimeboxMetrics` Represents measured stats metrics for timeboxes. @@ -17618,9 +17962,11 @@ Represents the Geo replication and verification state of an upload. | <a id="uploadregistryid"></a>`id` | [`ID!`](#id) | ID of the UploadRegistry. | | <a id="uploadregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the UploadRegistry. | | <a id="uploadregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the UploadRegistry. | -| <a id="uploadregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the UploadRegistry should be resynced. | +| <a id="uploadregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the UploadRegistry is resynced. | | <a id="uploadregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the UploadRegistry. | | <a id="uploadregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the UploadRegistry. | +| <a id="uploadregistryverificationretryat"></a>`verificationRetryAt` | [`Time`](#time) | Timestamp after which the UploadRegistry is reverified. | +| <a id="uploadregistryverifiedat"></a>`verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the UploadRegistry. | ### `UsageTrendsMeasurement` @@ -18510,14 +18856,19 @@ Represents vulnerability letter grades with associated projects. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="workitemclosedat"></a>`closedAt` | [`Time`](#time) | Timestamp of when the work item was closed. | +| <a id="workitemconfidential"></a>`confidential` | [`Boolean!`](#boolean) | Indicates the work item is confidential. | +| <a id="workitemcreatedat"></a>`createdAt` | [`Time!`](#time) | Timestamp of when the work item was created. | | <a id="workitemdescription"></a>`description` | [`String`](#string) | Description of the work item. | | <a id="workitemdescriptionhtml"></a>`descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | | <a id="workitemid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | | <a id="workitemiid"></a>`iid` | [`ID!`](#id) | Internal ID of the work item. | | <a id="workitemlockversion"></a>`lockVersion` | [`Int!`](#int) | Lock version of the work item. Incremented each time the work item is updated. | +| <a id="workitemproject"></a>`project` **{warning-solid}** | [`Project!`](#project) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Project the work item belongs to. | | <a id="workitemstate"></a>`state` | [`WorkItemState!`](#workitemstate) | State of the work item. | | <a id="workitemtitle"></a>`title` | [`String!`](#string) | Title of the work item. | | <a id="workitemtitlehtml"></a>`titleHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `title`. | +| <a id="workitemupdatedat"></a>`updatedAt` | [`Time!`](#time) | Timestamp of when the work item was last updated. | | <a id="workitemuserpermissions"></a>`userPermissions` | [`WorkItemPermissions!`](#workitempermissions) | Permissions for the current user on the resource. | | <a id="workitemwidgets"></a>`widgets` | [`[WorkItemWidget!]`](#workitemwidget) | Collection of widgets that belong to the work item. | | <a id="workitemworkitemtype"></a>`workItemType` | [`WorkItemType!`](#workitemtype) | Type assigned to the work item. | @@ -18581,6 +18932,41 @@ Represents a hierarchy widget. | <a id="workitemwidgethierarchyparent"></a>`parent` | [`WorkItem`](#workitem) | Parent work item. | | <a id="workitemwidgethierarchytype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetLabels` + +Represents the labels widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetlabelsallowsscopedlabels"></a>`allowsScopedLabels` | [`Boolean`](#boolean) | Indicates whether a scoped label is allowed. | +| <a id="workitemwidgetlabelslabels"></a>`labels` | [`LabelConnection`](#labelconnection) | Labels assigned to the work item. (see [Connections](#connections)) | +| <a id="workitemwidgetlabelstype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + +### `WorkItemWidgetStartAndDueDate` + +Represents a start and due date widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetstartandduedateduedate"></a>`dueDate` | [`Date`](#date) | Due date of the work item. | +| <a id="workitemwidgetstartandduedatestartdate"></a>`startDate` | [`Date`](#date) | Start date of the work item. | +| <a id="workitemwidgetstartandduedatetype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + +### `WorkItemWidgetVerificationStatus` + +Represents a verification status widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetverificationstatustype"></a>`type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +| <a id="workitemwidgetverificationstatusverificationstatus"></a>`verificationStatus` | [`String`](#string) | Verification status of the work item. | + ### `WorkItemWidgetWeight` Represents a weight widget. @@ -18853,15 +19239,14 @@ Values for sorting runners. | <a id="cirunnertypeinstance_type"></a>`INSTANCE_TYPE` | A runner that is instance type. | | <a id="cirunnertypeproject_type"></a>`PROJECT_TYPE` | A runner that is project type. | -### `CiRunnerUpgradeStatusType` +### `CiRunnerUpgradeStatus` | Value | Description | | ----- | ----------- | -| <a id="cirunnerupgradestatustypeavailable"></a>`AVAILABLE` | Upgrade is available for the runner. | -| <a id="cirunnerupgradestatustypeinvalid"></a>`INVALID` | Runner version is not valid. | -| <a id="cirunnerupgradestatustypenot_available"></a>`NOT_AVAILABLE` | Upgrade is not available for the runner. | -| <a id="cirunnerupgradestatustyperecommended"></a>`RECOMMENDED` | Upgrade is available and recommended for the runner. | -| <a id="cirunnerupgradestatustypeunknown"></a>`UNKNOWN` | Upgrade status is unknown. | +| <a id="cirunnerupgradestatusavailable"></a>`AVAILABLE` | Upgrade is available for the runner. | +| <a id="cirunnerupgradestatusinvalid"></a>`INVALID` | Runner version is not valid. | +| <a id="cirunnerupgradestatusnot_available"></a>`NOT_AVAILABLE` | Upgrade is not available for the runner. | +| <a id="cirunnerupgradestatusrecommended"></a>`RECOMMENDED` | Upgrade is available and recommended for the runner. | ### `CiVariableType` @@ -18946,6 +19331,33 @@ Conan file types. | <a id="conanmetadatumfiletypeenumpackage_file"></a>`PACKAGE_FILE` | A package file type. | | <a id="conanmetadatumfiletypeenumrecipe_file"></a>`RECIPE_FILE` | A recipe file type. | +### `ContactSort` + +Values for sorting contacts. + +| Value | Description | +| ----- | ----------- | +| <a id="contactsortcreated_asc"></a>`CREATED_ASC` | Created at ascending order. | +| <a id="contactsortcreated_desc"></a>`CREATED_DESC` | Created at descending order. | +| <a id="contactsortdescription_asc"></a>`DESCRIPTION_ASC` | Description by ascending order. | +| <a id="contactsortdescription_desc"></a>`DESCRIPTION_DESC` | Description by descending order. | +| <a id="contactsortemail_asc"></a>`EMAIL_ASC` | Email by ascending order. | +| <a id="contactsortemail_desc"></a>`EMAIL_DESC` | Email by descending order. | +| <a id="contactsortfirst_name_asc"></a>`FIRST_NAME_ASC` | First name by ascending order. | +| <a id="contactsortfirst_name_desc"></a>`FIRST_NAME_DESC` | First name by descending order. | +| <a id="contactsortlast_name_asc"></a>`LAST_NAME_ASC` | Last name by ascending order. | +| <a id="contactsortlast_name_desc"></a>`LAST_NAME_DESC` | Last name by descending order. | +| <a id="contactsortorganization_asc"></a>`ORGANIZATION_ASC` | Organization by ascending order. | +| <a id="contactsortorganization_desc"></a>`ORGANIZATION_DESC` | Organization by descending order. | +| <a id="contactsortphone_asc"></a>`PHONE_ASC` | Phone by ascending order. | +| <a id="contactsortphone_desc"></a>`PHONE_DESC` | Phone by descending order. | +| <a id="contactsortupdated_asc"></a>`UPDATED_ASC` | Updated at ascending order. | +| <a id="contactsortupdated_desc"></a>`UPDATED_DESC` | Updated at descending order. | +| <a id="contactsortcreated_asc"></a>`created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| <a id="contactsortcreated_desc"></a>`created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| <a id="contactsortupdated_asc"></a>`updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| <a id="contactsortupdated_desc"></a>`updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | + ### `ContainerExpirationPolicyCadenceEnum` | Value | Description | @@ -19027,8 +19439,9 @@ Values for sorting tags. | Value | Description | | ----- | ----------- | -| <a id="customerrelationscontactstateactive"></a>`active` | Active contact. | -| <a id="customerrelationscontactstateinactive"></a>`inactive` | Inactive contact. | +| <a id="customerrelationscontactstateactive"></a>`active` | Active contacts. | +| <a id="customerrelationscontactstateall"></a>`all` | All available contacts. | +| <a id="customerrelationscontactstateinactive"></a>`inactive` | Inactive contacts. | ### `CustomerRelationsOrganizationState` @@ -19172,6 +19585,24 @@ Mutation event of a design within a version. | <a id="designversioneventmodification"></a>`MODIFICATION` | A modification event. | | <a id="designversioneventnone"></a>`NONE` | No change. | +### `DetailedMergeStatus` + +Detailed representation of whether a GitLab merge request can be merged. + +| Value | Description | +| ----- | ----------- | +| <a id="detailedmergestatusblocked_status"></a>`BLOCKED_STATUS` | Merge request is blocked by another merge request. | +| <a id="detailedmergestatusbroken_status"></a>`BROKEN_STATUS` | Can not merge the source into the target branch, potential conflict. | +| <a id="detailedmergestatuschecking"></a>`CHECKING` | Currently checking for mergeability. | +| <a id="detailedmergestatusci_must_pass"></a>`CI_MUST_PASS` | Pipeline must succeed before merging. | +| <a id="detailedmergestatusdiscussions_not_resolved"></a>`DISCUSSIONS_NOT_RESOLVED` | Discussions must be resolved before merging. | +| <a id="detailedmergestatusdraft_status"></a>`DRAFT_STATUS` | Merge request must not be draft before merging. | +| <a id="detailedmergestatusmergeable"></a>`MERGEABLE` | Branch can be merged. | +| <a id="detailedmergestatusnot_approved"></a>`NOT_APPROVED` | Merge request must be approved before merging. | +| <a id="detailedmergestatusnot_open"></a>`NOT_OPEN` | Merge request must be open before merging. | +| <a id="detailedmergestatuspolicies_denied"></a>`POLICIES_DENIED` | There are denied policies for the merge request. | +| <a id="detailedmergestatusunchecked"></a>`UNCHECKED` | Merge status has not been checked. | + ### `DiffPositionType` Type of file the position refers to. @@ -19451,7 +19882,7 @@ Issue type. | <a id="issuetypeincident"></a>`INCIDENT` | Incident issue type. | | <a id="issuetypeissue"></a>`ISSUE` | Issue issue type. | | <a id="issuetyperequirement"></a>`REQUIREMENT` | Requirement issue type. | -| <a id="issuetypetask"></a>`TASK` **{warning-solid}** | **Introduced** in 15.2. This feature is in Alpha. It can be changed or removed at any time. | +| <a id="issuetypetask"></a>`TASK` **{warning-solid}** | **Introduced** in 15.2. This feature is in Alpha. It can be changed or removed at any time. Task issue type. Available only when feature flag `work_items` is enabled. | | <a id="issuetypetest_case"></a>`TEST_CASE` | Test Case issue type. | ### `IterationSearchableField` @@ -19508,6 +19939,7 @@ Iteration ID wildcard values. | <a id="jobartifactfiletypecodequality"></a>`CODEQUALITY` | CODE QUALITY job artifact file type. | | <a id="jobartifactfiletypecontainer_scanning"></a>`CONTAINER_SCANNING` | CONTAINER SCANNING job artifact file type. | | <a id="jobartifactfiletypecoverage_fuzzing"></a>`COVERAGE_FUZZING` | COVERAGE FUZZING job artifact file type. | +| <a id="jobartifactfiletypecyclonedx"></a>`CYCLONEDX` | CYCLONEDX job artifact file type. | | <a id="jobartifactfiletypedast"></a>`DAST` | DAST job artifact file type. | | <a id="jobartifactfiletypedependency_scanning"></a>`DEPENDENCY_SCANNING` | DEPENDENCY SCANNING job artifact file type. | | <a id="jobartifactfiletypedotenv"></a>`DOTENV` | DOTENV job artifact file type. | @@ -19917,6 +20349,15 @@ Release tag ID wildcard values. | <a id="releasetagwildcardidany"></a>`ANY` | Release tag is assigned. | | <a id="releasetagwildcardidnone"></a>`NONE` | No release tag is assigned. | +### `ReplicationStateEnum` + +| Value | Description | +| ----- | ----------- | +| <a id="replicationstateenumfailed"></a>`FAILED` | Replication process finished but failed. | +| <a id="replicationstateenumpending"></a>`PENDING` | Replication process has not started. | +| <a id="replicationstateenumstarted"></a>`STARTED` | Replication process is in progress. | +| <a id="replicationstateenumsynced"></a>`SYNCED` | Replication process finished successfully. | + ### `RequirementState` State of a requirement. @@ -20051,6 +20492,7 @@ State of a Sentry error. | <a id="servicetypepipelines_email_service"></a>`PIPELINES_EMAIL_SERVICE` | PipelinesEmailService type. | | <a id="servicetypepivotaltracker_service"></a>`PIVOTALTRACKER_SERVICE` | PivotaltrackerService type. | | <a id="servicetypeprometheus_service"></a>`PROMETHEUS_SERVICE` | PrometheusService type. | +| <a id="servicetypepumble_service"></a>`PUMBLE_SERVICE` | PumbleService type. | | <a id="servicetypepushover_service"></a>`PUSHOVER_SERVICE` | PushoverService type. | | <a id="servicetyperedmine_service"></a>`REDMINE_SERVICE` | RedmineService type. | | <a id="servicetypeshimo_service"></a>`SHIMO_SERVICE` | ShimoService type. | @@ -20217,6 +20659,10 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumgke_cluster_integration"></a>`GKE_CLUSTER_INTEGRATION` | Callout feature name for gke_cluster_integration. | | <a id="usercalloutfeaturenameenumgold_trial_billings"></a>`GOLD_TRIAL_BILLINGS` | Callout feature name for gold_trial_billings. | | <a id="usercalloutfeaturenameenummr_experience_survey"></a>`MR_EXPERIENCE_SURVEY` | Callout feature name for mr_experience_survey. | +| <a id="usercalloutfeaturenameenumnamespace_storage_limit_banner_alert_threshold"></a>`NAMESPACE_STORAGE_LIMIT_BANNER_ALERT_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_alert_threshold. | +| <a id="usercalloutfeaturenameenumnamespace_storage_limit_banner_error_threshold"></a>`NAMESPACE_STORAGE_LIMIT_BANNER_ERROR_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_error_threshold. | +| <a id="usercalloutfeaturenameenumnamespace_storage_limit_banner_info_threshold"></a>`NAMESPACE_STORAGE_LIMIT_BANNER_INFO_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_info_threshold. | +| <a id="usercalloutfeaturenameenumnamespace_storage_limit_banner_warning_threshold"></a>`NAMESPACE_STORAGE_LIMIT_BANNER_WARNING_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_warning_threshold. | | <a id="usercalloutfeaturenameenumnew_user_signups_cap_reached"></a>`NEW_USER_SIGNUPS_CAP_REACHED` | Callout feature name for new_user_signups_cap_reached. | | <a id="usercalloutfeaturenameenumpersonal_access_token_expiry"></a>`PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for personal_access_token_expiry. | | <a id="usercalloutfeaturenameenumpersonal_project_limitations_banner"></a>`PERSONAL_PROJECT_LIMITATIONS_BANNER` | Callout feature name for personal_project_limitations_banner. | @@ -20224,6 +20670,7 @@ Name of the feature that the callout is for. | <a id="usercalloutfeaturenameenumpipeline_needs_hover_tip"></a>`PIPELINE_NEEDS_HOVER_TIP` | Callout feature name for pipeline_needs_hover_tip. | | <a id="usercalloutfeaturenameenumpreview_user_over_limit_free_plan_alert"></a>`PREVIEW_USER_OVER_LIMIT_FREE_PLAN_ALERT` | Callout feature name for preview_user_over_limit_free_plan_alert. | | <a id="usercalloutfeaturenameenumprofile_personal_access_token_expiry"></a>`PROFILE_PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for profile_personal_access_token_expiry. | +| <a id="usercalloutfeaturenameenumproject_quality_summary_feedback"></a>`PROJECT_QUALITY_SUMMARY_FEEDBACK` | Callout feature name for project_quality_summary_feedback. | | <a id="usercalloutfeaturenameenumregistration_enabled_callout"></a>`REGISTRATION_ENABLED_CALLOUT` | Callout feature name for registration_enabled_callout. | | <a id="usercalloutfeaturenameenumsecurity_configuration_devops_alert"></a>`SECURITY_CONFIGURATION_DEVOPS_ALERT` | Callout feature name for security_configuration_devops_alert. | | <a id="usercalloutfeaturenameenumsecurity_configuration_upgrade_banner"></a>`SECURITY_CONFIGURATION_UPGRADE_BANNER` | Callout feature name for security_configuration_upgrade_banner. | @@ -20259,6 +20706,16 @@ Possible states of a user. | <a id="userstateblocked"></a>`blocked` | User has been blocked and is prevented from using the system. | | <a id="userstatedeactivated"></a>`deactivated` | User is no longer active and is unable to use the system. | +### `VerificationStateEnum` + +| Value | Description | +| ----- | ----------- | +| <a id="verificationstateenumdisabled"></a>`DISABLED` | Verification process is disabled. | +| <a id="verificationstateenumfailed"></a>`FAILED` | Verification process finished but failed. | +| <a id="verificationstateenumpending"></a>`PENDING` | Verification process has not started. | +| <a id="verificationstateenumstarted"></a>`STARTED` | Verification process is in progress. | +| <a id="verificationstateenumsucceeded"></a>`SUCCEEDED` | Verification process finished successfully. | + ### `VisibilityLevelsEnum` | Value | Description | @@ -20375,12 +20832,8 @@ Vulnerability sort values. | ----- | ----------- | | <a id="vulnerabilitysortdetected_asc"></a>`detected_asc` | Detection timestamp in ascending order. | | <a id="vulnerabilitysortdetected_desc"></a>`detected_desc` | Detection timestamp in descending order. | -| <a id="vulnerabilitysortreport_type_asc"></a>`report_type_asc` | Report Type in ascending order. | -| <a id="vulnerabilitysortreport_type_desc"></a>`report_type_desc` | Report Type in descending order. | | <a id="vulnerabilitysortseverity_asc"></a>`severity_asc` | Severity in ascending order. | | <a id="vulnerabilitysortseverity_desc"></a>`severity_desc` | Severity in descending order. | -| <a id="vulnerabilitysortstate_asc"></a>`state_asc` | State in ascending order. | -| <a id="vulnerabilitysortstate_desc"></a>`state_desc` | State in descending order. | ### `VulnerabilityState` @@ -20446,6 +20899,9 @@ Type of a work item widget. | <a id="workitemwidgettypeassignees"></a>`ASSIGNEES` | Assignees widget. | | <a id="workitemwidgettypedescription"></a>`DESCRIPTION` | Description widget. | | <a id="workitemwidgettypehierarchy"></a>`HIERARCHY` | Hierarchy widget. | +| <a id="workitemwidgettypelabels"></a>`LABELS` | Labels widget. | +| <a id="workitemwidgettypestart_and_due_date"></a>`START_AND_DUE_DATE` | Start And Due Date widget. | +| <a id="workitemwidgettypeverification_status"></a>`VERIFICATION_STATUS` | Verification Status widget. | | <a id="workitemwidgettypeweight"></a>`WEIGHT` | Weight widget. | ## Scalar types @@ -21008,6 +21464,12 @@ A regexp containing patterns sourced from user input. ### `Upload` +### `UploadID` + +A `UploadID` is a global ID. It is encoded as a string. + +An example `UploadID` is: `"gid://gitlab/Upload/1"`. + ### `UserID` A `UserID` is a global ID. It is encoded as a string. @@ -21189,6 +21651,25 @@ Implementations: | <a id="alertmanagementintegrationtype"></a>`type` | [`AlertManagementIntegrationType!`](#alertmanagementintegrationtype) | Type of integration. | | <a id="alertmanagementintegrationurl"></a>`url` | [`String`](#string) | Endpoint which accepts alert notifications. | +#### `CiVariable` + +Implementations: + +- [`CiGroupVariable`](#cigroupvariable) +- [`CiInstanceVariable`](#ciinstancevariable) +- [`CiManualVariable`](#cimanualvariable) +- [`CiProjectVariable`](#ciprojectvariable) + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="civariableid"></a>`id` | [`ID!`](#id) | ID of the variable. | +| <a id="civariablekey"></a>`key` | [`String`](#string) | Name of the variable. | +| <a id="civariableraw"></a>`raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| <a id="civariablevalue"></a>`value` | [`String`](#string) | Value of the variable. | +| <a id="civariablevariabletype"></a>`variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + #### `CurrentUserTodos` Implementations: @@ -21674,6 +22155,9 @@ Implementations: - [`WorkItemWidgetAssignees`](#workitemwidgetassignees) - [`WorkItemWidgetDescription`](#workitemwidgetdescription) - [`WorkItemWidgetHierarchy`](#workitemwidgethierarchy) +- [`WorkItemWidgetLabels`](#workitemwidgetlabels) +- [`WorkItemWidgetStartAndDueDate`](#workitemwidgetstartandduedate) +- [`WorkItemWidgetVerificationStatus`](#workitemwidgetverificationstatus) - [`WorkItemWidgetWeight`](#workitemwidgetweight) ##### Fields @@ -21731,6 +22215,17 @@ Field that are available while modifying the custom mapping attributes for an HT | <a id="boardissueinputweight"></a>`weight` | [`String`](#string) | Filter by weight. | | <a id="boardissueinputweightwildcardid"></a>`weightWildcardId` | [`WeightWildcardId`](#weightwildcardid) | Filter by weight ID wildcard. Incompatible with weight. | +### `CiVariableInput` + +Attributes for defining a CI/CD variable. + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="civariableinputkey"></a>`key` | [`String!`](#string) | Name of the variable. | +| <a id="civariableinputvalue"></a>`value` | [`String!`](#string) | Value of the variable. | + ### `CommitAction` #### Arguments @@ -21799,11 +22294,11 @@ Input type for DastSiteProfile authentication. | Name | Type | Description | | ---- | ---- | ----------- | | <a id="dastsiteprofileauthinputenabled"></a>`enabled` | [`Boolean`](#boolean) | Indicates whether authentication is enabled. | -| <a id="dastsiteprofileauthinputpassword"></a>`password` | [`String`](#string) | Password to authenticate with on the target website. | +| <a id="dastsiteprofileauthinputpassword"></a>`password` | [`String`](#string) | Password to authenticate with on the target. | | <a id="dastsiteprofileauthinputpasswordfield"></a>`passwordField` | [`String`](#string) | Name of password field at the sign-in HTML form. | | <a id="dastsiteprofileauthinputsubmitfield"></a>`submitField` | [`String`](#string) | Name or ID of sign-in submit button at the sign-in HTML form. | | <a id="dastsiteprofileauthinputurl"></a>`url` | [`String`](#string) | The URL of the page containing the sign-in HTML form on the target website. | -| <a id="dastsiteprofileauthinputusername"></a>`username` | [`String`](#string) | Username to authenticate with on the target website. | +| <a id="dastsiteprofileauthinputusername"></a>`username` | [`String`](#string) | Username to authenticate with on the target. | | <a id="dastsiteprofileauthinputusernamefield"></a>`usernameField` | [`String`](#string) | Name of username field at the sign-in HTML form. | ### `DiffImagePositionInput` @@ -22160,12 +22655,22 @@ A time-frame defined as a closed inclusive range of two dates. | Name | Type | Description | | ---- | ---- | ----------- | +| <a id="workitemupdatedtaskinputassigneeswidget"></a>`assigneesWidget` | [`WorkItemWidgetAssigneesInput`](#workitemwidgetassigneesinput) | Input for assignees widget. | +| <a id="workitemupdatedtaskinputconfidential"></a>`confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | | <a id="workitemupdatedtaskinputdescriptionwidget"></a>`descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | | <a id="workitemupdatedtaskinputhierarchywidget"></a>`hierarchyWidget` | [`WorkItemWidgetHierarchyUpdateInput`](#workitemwidgethierarchyupdateinput) | Input for hierarchy widget. | | <a id="workitemupdatedtaskinputid"></a>`id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| <a id="workitemupdatedtaskinputstartandduedatewidget"></a>`startAndDueDateWidget` | [`WorkItemWidgetStartAndDueDateUpdateInput`](#workitemwidgetstartandduedateupdateinput) | Input for start and due date widget. | | <a id="workitemupdatedtaskinputstateevent"></a>`stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | | <a id="workitemupdatedtaskinputtitle"></a>`title` | [`String`](#string) | Title of the work item. | -| <a id="workitemupdatedtaskinputweightwidget"></a>`weightWidget` | [`WorkItemWidgetWeightInput`](#workitemwidgetweightinput) | Input for weight widget. | + +### `WorkItemWidgetAssigneesInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetassigneesinputassigneeids"></a>`assigneeIds` | [`[UserID!]!`](#userid) | Global IDs of assignees. | ### `WorkItemWidgetDescriptionInput` @@ -22192,10 +22697,19 @@ A time-frame defined as a closed inclusive range of two dates. | <a id="workitemwidgethierarchyupdateinputchildrenids"></a>`childrenIds` | [`[WorkItemID!]`](#workitemid) | Global IDs of children work items. | | <a id="workitemwidgethierarchyupdateinputparentid"></a>`parentId` | [`WorkItemID`](#workitemid) | Global ID of the parent work item. Use `null` to remove the association. | +### `WorkItemWidgetStartAndDueDateUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| <a id="workitemwidgetstartandduedateupdateinputduedate"></a>`dueDate` | [`Date`](#date) | Due date for the work item. | +| <a id="workitemwidgetstartandduedateupdateinputstartdate"></a>`startDate` | [`Date`](#date) | Start date for the work item. | + ### `WorkItemWidgetWeightInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| <a id="workitemwidgetweightinputweight"></a>`weight` | [`Int!`](#int) | Weight of the work item. | +| <a id="workitemwidgetweightinputweight"></a>`weight` | [`Int`](#int) | Weight of the work item. | diff --git a/doc/api/group_protected_environments.md b/doc/api/group_protected_environments.md index 8ebd0dcd99a..0f1527f8968 100644 --- a/doc/api/group_protected_environments.md +++ b/doc/api/group_protected_environments.md @@ -106,7 +106,7 @@ POST /groups/:id/protected_environments | `id` | integer/string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) maintained by the authenticated user. | | `name` | string | yes | The deployment tier of the protected environment. One of `production`, `staging`, `testing`, `development`, or `other`. Read more about [deployment tiers](../ci/environments/index.md#deployment-tier-of-environments).| | `deploy_access_levels` | array | yes | Array of access levels allowed to deploy, with each described by a hash. One of `user_id`, `group_id` or `access_level`. They take the form of `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}` respectively. | -| `required_approval_count` | integer | no | The number of approvals required to deploy to this environment. This is part of Deployment Approvals, which isn't yet available for use. For details, see [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343864). | +| `required_approval_count` | integer | no | The number of approvals required to deploy to this environment. | | `approval_rules` | array | no | Array of access levels allowed to approve, with each described by a hash. One of `user_id`, `group_id` or `access_level`. They take the form of `{user_id: integer}`, `{group_id: integer}` or `{access_level: integer}` respectively. You can also specify the number of required approvals from the specified entity with `required_approvals` field. See [Multiple approval rules](../ci/environments/deployment_approvals.md#multiple-approval-rules) for more information. | The assignable `user_id` are the users who belong to the given group with the Maintainer role (or above). diff --git a/doc/api/groups.md b/doc/api/groups.md index c51f23decb9..588ab2a5821 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -452,6 +452,7 @@ Example response: "open_issues_count":10, "ci_default_git_depth":50, "ci_forward_deployment_enabled":true, + "ci_allow_fork_pipelines_to_run_in_parent_project":true, "public_jobs":true, "build_timeout":3600, "auto_cancel_pending_pipelines":"enabled", @@ -901,6 +902,13 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ ## Update group +> `unique_project_download_limit`, `unique_project_download_limit_interval_in_seconds`, and `unique_project_download_limit_allowlist` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92970) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `limit_unique_project_downloads_per_namespace_user`. Disabled by default. + +FLAG: +On self-managed GitLab, by default `unique_project_download_limit`, `unique_project_download_limit_interval_in_seconds`, and `unique_project_download_limit_allowlist` are not available. +To make them available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `limit_unique_project_downloads_per_namespace_user`. + Updates the project group. Only available to group owners and administrators. ```plaintext @@ -919,7 +927,7 @@ PUT /groups/:id | `emails_disabled` | boolean | no | Disable email notifications. | | `lfs_enabled` | boolean | no | Enable/disable Large File Storage (LFS) for the projects in this group. | | `mentions_disabled` | boolean | no | Disable the capability of a group from getting mentioned. | -| `prevent_sharing_groups_outside_hierarchy` | boolean | no | See [Prevent group sharing outside the group hierarchy](../user/group/index.md#prevent-group-sharing-outside-the-group-hierarchy). This attribute is only available on top-level groups. [Introduced in GitLab 14.1](https://gitlab.com/gitlab-org/gitlab/-/issues/333721) | +| `prevent_sharing_groups_outside_hierarchy` | boolean | no | See [Prevent group sharing outside the group hierarchy](../user/group/access_and_permissions.md#prevent-group-sharing-outside-the-group-hierarchy). This attribute is only available on top-level groups. [Introduced in GitLab 14.1](https://gitlab.com/gitlab-org/gitlab/-/issues/333721) | | `project_creation_level` | string | no | Determine if developers can create projects in the group. Can be `noone` (No one), `maintainer` (users with the Maintainer role), or `developer` (users with the Developer or Maintainer role). | | `request_access_enabled` | boolean | no | Allow users to request member access. | | `require_two_factor_authentication` | boolean | no | Require all users in this group to setup Two-factor authentication. | @@ -933,6 +941,9 @@ PUT /groups/:id | `membership_lock` **(PREMIUM)** | boolean | no | Users cannot be added to projects in this group. | | `prevent_forking_outside_group` **(PREMIUM)** | boolean | no | When enabled, users can **not** fork projects from this group to external namespaces. | | `shared_runners_minutes_limit` **(PREMIUM)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | +| `unique_project_download_limit` **(ULTIMATE)** | integer | no | Maximum number of unique projects a user can download in the specified time period before they are banned. Available only on top-level groups. Default: 0, Maximum: 10,000. | +| `unique_project_download_limit_interval_in_seconds` **(ULTIMATE)** | integer | no | Time period during which a user can download a maximum amount of projects before they are banned. Available only on top-level groups. Default: 0, Maximum: 864,000 seconds (10 days). | +| `unique_project_download_limit_allowlist` **(ULTIMATE)** | array of strings | no | List of usernames excluded from the unique project download limit. Available only on top-level groups. Default: `[]`, Maximum: 100 usernames. | NOTE: The `projects` and `shared_projects` attributes in the response are deprecated and [scheduled for removal in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). @@ -1407,6 +1418,155 @@ DELETE /groups/:id/ldap_group_links NOTE: To delete the LDAP group link, provide either a `cn` or a `filter`, but not both. +## SAML Group Links **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/290367) in GitLab 15.3. + +List, get, add, and delete SAML group links. + +### List SAML group links + +Lists SAML group links. + +```plaintext +GET /groups/:id/saml_group_links +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|:----------|:---------------|:---------|:-------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | + +If successful, returns [`200`](index.md#status-codes) and the following +response attributes: + +| Attribute | Type | Description | +|:-------------------|:-------|:-------------------------------------------------------------------------------------| +| `[].name` | string | Name of the SAML group | +| `[].access_level` | string | Minimum [access level](members.md#valid-access-levels) for members of the SAML group | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/saml_group_links" +``` + +Example response: + +```json +[ + { + "name": "saml-group-1", + "access_level": "Guest" + }, + { + "name": "saml-group-2", + "access_level": "Maintainer" + } +] +``` + +### Get SAML group link + +Get a SAML group link for the group. + +```plaintext +GET /groups/:id/saml_group_links/:saml_group_name +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|:-------------------|:---------------|:---------|:-------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | +| `saml_group_name` | string | yes | Name of an SAML group | + +If successful, returns [`200`](index.md#status-codes) and the following +response attributes: + +| Attribute | Type | Description | +|:---------------|:-------|:-------------------------------------------------------------------------------------| +| `name` | string | Name of the SAML group | +| `access_level` | string | Minimum [access level](members.md#valid-access-levels) for members of the SAML group | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/saml_group_links/saml-group-1" +``` + +Example response: + +```json +{ +"name": "saml-group-1", +"access_level": "Guest" +} +``` + +### Add SAML group link + +Adds a SAML group link for a group. + +```plaintext +POST /groups/:id/saml_group_links +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|:-------------------|:---------------|:---------|:-------------------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | +| `saml_group_name` | string | yes | Name of a SAML group | +| `access_level` | string | yes | Minimum [access level](members.md#valid-access-levels) for members of the SAML group | + +If successful, returns [`201`](index.md#status-codes) and the following +response attributes: + +| Attribute | Type | Description | +|:---------------|:-------|:-------------------------------------------------------------------------------------| +| `name` | string | Name of the SAML group | +| `access_level` | string | Minimum [access level](members.md#valid-access-levels) for members of the SAML group | + +Example request: + +```shell +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/saml_group_links" +``` + +Example response: + +```json +{ +"name": "saml-group-1", +"access_level": "Guest" +} +``` + +### Delete SAML group link + +Deletes a SAML group link for the group. + +```plaintext +DELETE /groups/:id/saml_group_links/:saml_group_name +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|:-------------------|:---------------|:---------|:-------------------------------------------------------------------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | +| `saml_group_name` | string | yes | Name of an SAML group | + +If successful, returns [`204`](index.md#status-codes) status code without any response body. + +Example request: + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/1/saml_group_links/saml-group-1" +``` + ## Namespaces in groups By default, groups only get 20 namespaces at a time because the API results are paginated. @@ -1435,7 +1595,7 @@ documentation. ## Share Groups with Groups -These endpoints create and delete links for sharing a group with another group. For more information, see the related discussion in the [GitLab Groups](../user/group/index.md#share-a-group-with-another-group) page. +These endpoints create and delete links for sharing a group with another group. For more information, see the related discussion in the [GitLab Groups](../user/group/manage.md#share-a-group-with-another-group) page. ### Create a link to share a group with another group @@ -1471,7 +1631,7 @@ DELETE /groups/:id/share/:group_id ### Get group push rules **(PREMIUM)** -Get the [push rules](../user/group/index.md#group-push-rules) of a group. +Get the [push rules](../user/group/access_and_permissions.md#group-push-rules) of a group. Only available to group owners and administrators. @@ -1514,7 +1674,7 @@ the `commit_committer_check` and `reject_unsigned_commits` parameters: ### Add group push rule **(PREMIUM)** -Adds [push rules](../user/group/index.md#group-push-rules) to the specified group. +Adds [push rules](../user/group/access_and_permissions.md#group-push-rules) to the specified group. Only available to group owners and administrators. @@ -1608,7 +1768,7 @@ Response: ### Delete group push rule **(PREMIUM)** -Deletes the [push rules](../user/group/index.md#group-push-rules) of a group. +Deletes the [push rules](../user/group/access_and_permissions.md#group-push-rules) of a group. Only available to group owners and administrators. diff --git a/doc/api/index.md b/doc/api/index.md index 26447a2223d..15d0b0fd65f 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -522,11 +522,11 @@ pagination headers. Keyset-based pagination is supported only for selected resources and ordering options: -| Resource | Options | Availability | -|:---------------------------------------------------------|:---------------------------------|:------------------------------------------------------------------------------------------------------------| -| [Projects](projects.md) | `order_by=id` only | Authenticated and unauthenticated users | -| [Groups](groups.md) | `order_by=name`, `sort=asc` only | Unauthenticated users only | -| [Group audit events](audit_events.md#group-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2 | +| Resource | Options | Availability | +|:---------------------------------------------------------|:---------------------------------|:-------------------------------------------------------------------------------------------------------------| +| [Projects](projects.md) | `order_by=id` only | Authenticated and unauthenticated users | +| [Groups](groups.md) | `order_by=name`, `sort=asc` only | Unauthenticated users only | +| [Group audit events](audit_events.md#group-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2) | ### Pagination response headers diff --git a/doc/api/integrations.md b/doc/api/integrations.md index fca1d02161b..7912f3f6bf7 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -310,17 +310,15 @@ PUT /projects/:id/integrations/datadog Parameters: -| Parameter | Type | Required | Description | -|:-----------------:|:------:|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `api_key` | string | true | API key used for authentication with Datadog. | -| `api_url` | string | false | (Advanced) The full URL for your Datadog site | -| `datadog_env` | string | false | For self-managed deployments, set the env% tag for all the data sent to Datadog. | -| `datadog_service` | string | false | Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments | -| `datadog_site` | string | false | The Datadog site to send data to. To send data to the EU site, use `datadoghq.eu` | -| `datadog_tags` | string | false | Custom tags in Datadog. Specify one tag per line in the format: `key:value\nkey2:value2` ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79665) in GitLab 14.8.) | - -<!-- | `archive_trace_events` | boolean | false | When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346339) in GitLab 14.7) | --> -<!-- TODO: uncomment the archive_trace_events field once :datadog_integration_logs_collection is rolled out. Rollout issue: https://gitlab.com/gitlab-org/gitlab/-/issues/346339 --> +| Parameter | Type | Required | Description | +|------------------------|---------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `api_key` | string | true | API key used for authentication with Datadog. | +| `api_url` | string | false | (Advanced) The full URL for your Datadog site | +| `datadog_env` | string | false | For self-managed deployments, set the env% tag for all the data sent to Datadog. | +| `datadog_service` | string | false | Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments | +| `datadog_site` | string | false | The Datadog site to send data to. To send data to the EU site, use `datadoghq.eu` | +| `datadog_tags` | string | false | Custom tags in Datadog. Specify one tag per line in the format: `key:value\nkey2:value2` ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79665) in GitLab 14.8.) | +| `archive_trace_events` | boolean | false | When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346339) in GitLab 15.3) | ### Disable Datadog integration @@ -383,6 +381,51 @@ Get Unify Circuit integration settings for a project. GET /projects/:id/integrations/unify-circuit ``` +## Pumble + +Pumble chat tool. + +### Create/Edit Pumble integration + +Set Pumble integration for a project. + +```plaintext +PUT /projects/:id/integrations/pumble +``` + +Parameters: + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `webhook` | string | true | The Pumble webhook. For example, `https://api.pumble.com/workspaces/x/...`. | +| `branches_to_be_notified` | string | false | Branches to send notifications for. Valid options are `all`, `default`, `protected`, and `default_and_protected`. The default is `default`. | +| `confidential_issues_events` | boolean | false | Enable notifications for confidential issue events. | +| `confidential_note_events` | boolean | false | Enable notifications for confidential note events. | +| `issues_events` | boolean | false | Enable notifications for issue events. | +| `merge_requests_events` | boolean | false | Enable notifications for merge request events. | +| `note_events` | boolean | false | Enable notifications for note events. | +| `notify_only_broken_pipelines` | boolean | false | Send notifications for broken pipelines. | +| `pipeline_events` | boolean | false | Enable notifications for pipeline events. | +| `push_events` | boolean | false | Enable notifications for push events. | +| `tag_push_events` | boolean | false | Enable notifications for tag push events. | +| `wiki_page_events` | boolean | false | Enable notifications for wiki page events. | + +### Disable Pumble integration + +Disable the Pumble integration for a project. Integration settings are preserved. + +```plaintext +DELETE /projects/:id/integrations/pumble +``` + +### Get Pumble integration settings + +Get Pumble integration settings for a project. + +```plaintext +GET /projects/:id/integrations/pumble +``` + ## Webex Teams Webex Teams collaboration tool. @@ -967,7 +1010,6 @@ Parameters: | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `token` | string | yes | The Mattermost token | -| `username` | string | no | The username to use to post the message | ### Disable Mattermost Slash Command integration diff --git a/doc/api/issues.md b/doc/api/issues.md index 1f5f4b4c8ae..e2c9fbd878d 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -29,9 +29,9 @@ When requested across groups or projects, it's expected to be the same as the `f ## List issues -> The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. -> The `weight` property moved to GitLab Premium in 13.9. -> The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. +> - The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. +> - The `weight` property moved to GitLab Premium in 13.9. +> - The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. Get all issues the authenticated user has access to. By default it returns only issues created by the current user. To get all issues, @@ -257,9 +257,9 @@ Please use `iid` of the `epic` attribute instead. ## List group issues -> The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. -> The `weight` property moved to GitLab Premium in 13.9. -> The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. +> - The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. +> - The `weight` property moved to GitLab Premium in 13.9. +> - The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. Get a list of a group's issues. @@ -461,9 +461,9 @@ Please use `iid` of the `epic` attribute instead. ## List project issues -> The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. -> The `weight` property moved to GitLab Premium in 13.9. -> The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. +> - The `due_date` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233420) in GitLab 13.3. +> - The `weight` property moved to GitLab Premium in 13.9. +> - The `due_date` filters `any`, `today`, and `tomorrow` were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78460) in GitLab 14.8. Get a list of a project's issues. diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index ee9f1678b18..31da0638d23 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -16,11 +16,11 @@ Get the job's artifacts zipped archive of a project. GET /projects/:id/jobs/:job_id/artifacts ``` -| Attribute | Type | Required | Description | -|-------------|----------------|----------|--------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `job_id` | integer | yes | ID of a job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| Attribute | Type | Required | Description | +|---------------------------|----------------|----------|-------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | ID of a job. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only in a CI/CD job defined in the `.gitlab-ci.yml` file. The value is always `$CI_JOB_TOKEN`. The job associated with the `$CI_JOB_TOKEN` must be running when this token is used. | Example request using the `PRIVATE-TOKEN` header: @@ -80,12 +80,12 @@ GET /projects/:id/jobs/artifacts/:ref_name/download?job=name Parameters -| Attribute | Type | Required | Description | -|-------------|----------------|----------|--------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | -| `job` | string | yes | The name of the job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| Attribute | Type | Required | Description | +|---------------------------|----------------|----------|-------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `ref_name` | string | yes | Branch or tag name in repository. HEAD or SHA references are not supported. | +| `job` | string | yes | The name of the job. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only in a CI/CD job defined in the `.gitlab-ci.yml` file. The value is always `$CI_JOB_TOKEN`. The job associated with the `$CI_JOB_TOKEN` must be running when this token is used. | Example request using the `PRIVATE-TOKEN` header: @@ -141,12 +141,12 @@ GET /projects/:id/jobs/:job_id/artifacts/*artifact_path Parameters -| Attribute | Type | Required | Description | -|-----------------|----------------|----------|------------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `job_id` | integer | yes | The unique job identifier. | -| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| Attribute | Type | Required | Description | +|---------------------------|----------------|----------|-------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `job_id` | integer | yes | The unique job identifier. | +| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only in a CI/CD job defined in the `.gitlab-ci.yml` file. The value is always `$CI_JOB_TOKEN`. The job associated with the `$CI_JOB_TOKEN` must be running when this token is used. | Example request: @@ -185,13 +185,13 @@ GET /projects/:id/jobs/artifacts/:ref_name/raw/*artifact_path?job=name Parameters: -| Attribute | Type | Required | Description | -|-----------------|----------------|----------|--------------------------------------------------------------------------------------------------------------| -| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `ref_name` | string | yes | Branch or tag name in repository. `HEAD` or `SHA` references are not supported. | -| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | -| `job` | string | yes | The name of the job. | -| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only inside `.gitlab-ci.yml`. Its value is always `$CI_JOB_TOKEN`. | +| Attribute | Type | Required | Description | +|---------------------------|----------------|----------|-------------| +| `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `ref_name` | string | yes | Branch or tag name in repository. `HEAD` or `SHA` references are not supported. | +| `artifact_path` | string | yes | Path to a file inside the artifacts archive. | +| `job` | string | yes | The name of the job. | +| `job_token` **(PREMIUM)** | string | no | To be used with [triggers](../ci/jobs/ci_job_token.md#download-an-artifact-from-a-different-pipeline) for multi-project pipelines. It should be invoked only in a CI/CD job defined in the `.gitlab-ci.yml` file. The value is always `$CI_JOB_TOKEN`. The job associated with the `$CI_JOB_TOKEN` must be running when this token is used. | Example request: diff --git a/doc/api/jobs.md b/doc/api/jobs.md index b23c33ddc0d..647f8eafa62 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -19,7 +19,7 @@ GET /projects/:id/jobs | Attribute | Type | Required | Description | |-----------|--------------------------------|------------------------|-------------| | `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | -| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | +| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, `waiting_for_resource`, or `manual`. All jobs are returned if `scope` is not provided. | ```shell curl --globoff --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/jobs?scope[]=pending&scope[]=running" @@ -167,7 +167,7 @@ GET /projects/:id/pipelines/:pipeline_id/jobs |-------------------|--------------------------------|------------------------|-------------| | `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | | `pipeline_id` | integer | **{check-circle}** Yes | ID of a pipeline. Can also be obtained in CI jobs via the [predefined CI variable](../ci/variables/predefined_variables.md) `CI_PIPELINE_ID`. | -| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | +| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, `waiting_for_resource`, or `manual`. All jobs are returned if `scope` is not provided. | | `include_retried` | boolean | **{dotted-circle}** No | Include retried jobs in the response. Defaults to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/272627) in GitLab 13.9. | ```shell @@ -324,7 +324,7 @@ GET /projects/:id/pipelines/:pipeline_id/bridges |---------------|--------------------------------|------------------------|-------------| | `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | | `pipeline_id` | integer | **{check-circle}** Yes | ID of a pipeline. | -| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, or `manual`. All jobs are returned if `scope` is not provided. | +| `scope` | string **or** array of strings | **{dotted-circle}** No | Scope of jobs to show. Either one of or an array of the following: `created`, `pending`, `running`, `failed`, `success`, `canceled`, `skipped`, `waiting_for_resource`, or `manual`. All jobs are returned if `scope` is not provided. | ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipelines/6/bridges?scope[]=pending&scope[]=running" @@ -700,7 +700,7 @@ Example of response "stage": "test", "status": "canceled", "tag": false, - "web_url": "https://example.com/foo/bar/-/jobs/42", + "web_url": "https://example.com/foo/bar/-/jobs/1", "user": null } ``` @@ -750,7 +750,7 @@ Example of response "stage": "test", "status": "pending", "tag": false, - "web_url": "https://example.com/foo/bar/-/jobs/42", + "web_url": "https://example.com/foo/bar/-/jobs/1", "user": null } ``` @@ -805,7 +805,7 @@ Example of response "queued_duration": 0.010, "status": "failed", "tag": false, - "web_url": "https://example.com/foo/bar/-/jobs/42", + "web_url": "https://example.com/foo/bar/-/jobs/1", "user": null } ``` @@ -881,7 +881,7 @@ Example response: "stage": "test", "status": "pending", "tag": false, - "web_url": "https://example.com/foo/bar/-/jobs/42", + "web_url": "https://example.com/foo/bar/-/jobs/1", "user": null } ``` diff --git a/doc/api/lint.md b/doc/api/lint.md index d5aa6af0e34..ff7aa2f1fb9 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -17,8 +17,7 @@ and: - Does not have an [allowlist or denylist](../user/admin_area/settings/sign_up_restrictions.md#allow-or-deny-sign-ups-using-specific-email-domains). - Does not [require administrator approval for new sign ups](../user/admin_area/settings/sign_up_restrictions.md#require-administrator-approval-for-new-sign-ups). -- Does not have additional [sign up - restrictions](../user/admin_area/settings/sign_up_restrictions.html#sign-up-restrictions). +- Does not have additional [sign up restrictions](../user/admin_area/settings/sign_up_restrictions.md). Otherwise, authentication is required. diff --git a/doc/api/markdown.md b/doc/api/markdown.md index c128e8512df..b66a07dc1d5 100644 --- a/doc/api/markdown.md +++ b/doc/api/markdown.md @@ -1,13 +1,27 @@ --- -stage: Create -group: Source Code +stage: Plan +group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Markdown API **(FREE)** +Convert Markdown content to HTML. + Available only in APIv4. +## Required authentication + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93727) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `authenticate_markdown_api`. Enabled by default. + +FLAG: +On self-managed GitLab, by default this feature is enabled and authentication is required. +To remove the requirement to authenticate, ask an administrator to +[disable the feature flag](../administration/feature_flags.md) named `authenticate_markdown_api`. +On GitLab.com, this feature is available. + +All API calls to the Markdown API must be [authenticated](index.md#authentication). + ## Render an arbitrary Markdown document ```plaintext @@ -18,10 +32,12 @@ POST /markdown | --------- | ------- | ------------- | ------------------------------------------ | | `text` | string | yes | The Markdown text to render | | `gfm` | boolean | no | Render text using GitLab Flavored Markdown. Default is `false` | -| `project` | string | no | Use `project` as a context when creating references using GitLab Flavored Markdown. [Authentication](index.md#authentication) is required if a project is not public. | +| `project` | string | no | Use `project` as a context when creating references using GitLab Flavored Markdown | ```shell -curl --header Content-Type:application/json --data '{"text":"Hello world! :tada:", "gfm":true, "project":"group_example/project_example"}' "https://gitlab.example.com/api/v4/markdown" +curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ + --header "Content-Type:application/json" \ + --data '{"text":"Hello world! :tada:", "gfm":true, "project":"group_example/project_example"}' "https://gitlab.example.com/api/v4/markdown" ``` Response example: diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 37a926366df..1d99c323946 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -79,6 +79,8 @@ POST /projects/:id/approvals > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. > - `protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.7. +> - Pagination support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31011) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `approval_rules_pagination`. Enabled by default. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can request information about a project's approval rules using the following endpoint: @@ -86,6 +88,8 @@ You can request information about a project's approval rules using the following GET /projects/:id/approval_rules ``` +Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) parameters to restrict the list of approval rules. + **Parameters:** | Attribute | Type | Required | Description | @@ -145,6 +149,7 @@ GET /projects/:id/approval_rules "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -177,7 +182,8 @@ GET /projects/:id/approval_rules ### Get a single project-level rule -> Introduced in GitLab 13.7. +> - Introduced in GitLab 13.7. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can request information about a single project approval rules using the following endpoint: @@ -244,6 +250,7 @@ GET /projects/:id/approval_rules/:approval_rule_id "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -278,6 +285,7 @@ GET /projects/:id/approval_rules/:approval_rule_id > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can create project approval rules using the following endpoint: @@ -287,16 +295,17 @@ POST /projects/:id/approval_rules **Parameters:** -| Attribute | Type | Required | Description | -|------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `name` | string | yes | The name of the approval rule | -| `approvals_required` | integer | yes | The number of required approvals for this rule | -| `rule_type` | string | no | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. -| `user_ids` | Array | no | The ids of users as approvers | -| `group_ids` | Array | no | The ids of groups as approvers | -| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `license_scanning` and `code_coverage`.| +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `name` | string | yes | The name of the approval rule | +| `report_type` | string | no | The report type required when the rule type is `report_approver`. The supported report types are: `license_scanning` and `code_coverage`. | +| `approvals_required` | integer | yes | The number of required approvals for this rule | +| `rule_type` | string | no | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | +| `user_ids` | Array | no | The ids of users as approvers | +| `group_ids` | Array | no | The ids of groups as approvers | +| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `applies_to_all_protected_branches` | boolean | no | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | ```json { @@ -350,6 +359,7 @@ POST /projects/:id/approval_rules "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -401,6 +411,7 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" \ > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11877) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. +> - `applies_to_all_protected_branches` property was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. You can update project approval rules using the following endpoint: @@ -412,15 +423,16 @@ PUT /projects/:id/approval_rules/:approval_rule_id **Parameters:** -| Attribute | Type | Required | Description | -|------------------------|---------|----------|------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `approval_rule_id` | integer | yes | The ID of a approval rule | -| `name` | string | yes | The name of the approval rule | -| `approvals_required` | integer | yes | The number of required approvals for this rule | -| `user_ids` | Array | no | The ids of users as approvers | -| `group_ids` | Array | no | The ids of groups as approvers | -| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `approval_rule_id` | integer | yes | The ID of a approval rule | +| `name` | string | yes | The name of the approval rule | +| `approvals_required` | integer | yes | The number of required approvals for this rule | +| `user_ids` | Array | no | The ids of users as approvers | +| `group_ids` | Array | no | The ids of groups as approvers | +| `protected_branch_ids` | Array | no | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `applies_to_all_protected_branches` | boolean | no | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | ```json { @@ -474,6 +486,7 @@ PUT /projects/:id/approval_rules/:approval_rule_id "ldap_access": null } ], + "applies_to_all_protected_branches": false, "protected_branches": [ { "id": 1, @@ -516,10 +529,10 @@ DELETE /projects/:id/approval_rules/:approval_rule_id **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `approval_rule_id` | integer | yes | The ID of a approval rule +| Attribute | Type | Required | Description | +|--------------------|-------------------|----------|------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `approval_rule_id` | integer | yes | The ID of a approval rule | ## Merge request-level MR approvals @@ -538,10 +551,10 @@ GET /projects/:id/merge_requests/:merge_request_iid/approvals **Parameters:** -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| +| Attribute | Type | Required | Description | +|---------------------|-------------------|----------|------------------------------------------------------------------------------| | `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | +| `merge_request_iid` | integer | yes | The IID of MR | ```json { @@ -584,11 +597,11 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|--------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | -| `approvals_required` | integer | yes | Approvals required before MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | +| Attribute | Type | Required | Description | +|----------------------|-------------------|----------|------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of MR | +| `approvals_required` | integer | yes | Approvals required before MR can be merged. Deprecated in 12.0 in favor of Approval Rules API. | ```json { @@ -626,10 +639,10 @@ This includes additional information about the users who have already approved **Parameters:** -| Attribute | Type | Required | Description | -|----------------------|---------|----------|---------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of MR | +| Attribute | Type | Required | Description | +|---------------------|-------------------|----------|------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `merge_request_iid` | integer | yes | The IID of MR | ```json { @@ -684,6 +697,7 @@ This includes additional information about the users who have already approved > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13712) in GitLab 12.3. > - Moved to GitLab Premium in 13.9. +> - Pagination support [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31011) in GitLab 15.3 [with a flag](../administration/feature_flags.md) named `approval_rules_pagination`. Enabled by default. You can request information about a merge request's approval rules using the following endpoint: @@ -691,6 +705,8 @@ You can request information about a merge request's approval rules using the fol GET /projects/:id/merge_requests/:merge_request_iid/approval_rules ``` +Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) parameters to restrict the list of approval rules. + **Parameters:** | Attribute | Type | Required | Description | diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index c6714459643..a491756e5f0 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -825,6 +825,66 @@ Parameters: ] ``` +## Get single MR reviewers + +Get a list of merge request reviewers. + +```plaintext +GET /projects/:id/merge_requests/:merge_request_iid/reviewers +``` + +Parameters: + +| Attribute | Type | Required | Description | +|---------------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `merge_request_iid` | integer | yes | The internal ID of the merge request. | + +```json +[ + { + "user": { + "id": 1, + "name": "John Doe1", + "username": "user1", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon", + "web_url": "http://localhost/user1" + }, + "updated_state_by": { + "id": 1, + "name": "John Doe1", + "username": "user1", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon", + "web_url": "http://localhost/user1" + }, + "state": "unreviewed", + "created_at": "2022-07-27T17:03:27.684Z" + }, + { + "user": { + "id": 2, + "name": "John Doe2", + "username": "user2", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/10fc7f102be8de7657fb4d80898bbfe3?s=80&d=identicon", + "web_url": "http://localhost/user2" + }, + "updated_state_by": { + "id": 1, + "name": "John Doe1", + "username": "user1", + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/c922747a93b40d1ea88262bf1aebee62?s=80&d=identicon", + "web_url": "http://localhost/user1" + }, + "state": "reviewed", + "created_at": "2022-07-27T17:03:27.684Z" + } +] +``` + ## Get single MR commits Get a list of merge request commits. @@ -1624,9 +1684,9 @@ This API returns specific HTTP status codes on failure: | HTTP Status | Message | Reason | |:------------|--------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| | `401` | `Unauthorized` | This user does not have permission to accept this merge request. | -| `405` | `Method Not Allowed` | The merge request cannot be accepted because it is `Draft`, `Closed`, `Pipeline Pending Completion`, or `Failed`. `Success` is required. | -| `406` | `Branch cannot be merged` | The merge request can not be merged. | +| `405` | `Method Not Allowed` | The merge request is not able to be merged. | | `409` | `SHA does not match HEAD of source branch` | The provided `sha` parameter does not match the HEAD of the source. | +| `422` | `Branch cannot be merged` | The merge request failed to merge. | For additional important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index 3972a46d7fc..50c97d55f45 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Namespaces API **(FREE)** Usernames and group names fall under a special category called -[namespaces](../user/group/index.md#namespaces). +[namespaces](../user/namespace/index.md). For users and groups supported API calls see the [users](users.md) and [groups](groups.md) documentation respectively. diff --git a/doc/api/notes.md b/doc/api/notes.md index f7caae59b4d..e0799cdd380 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -11,7 +11,7 @@ Notes are comments on: - Snippets - Issues - Merge requests -- Epics **(PREMIUM)** +- [Epics](../user/group/epics/index.md) This includes system notes, which are notes about changes to the object (for example, when an assignee changes, GitLab posts a system note). @@ -80,7 +80,8 @@ GET /projects/:id/issues/:issue_iid/notes?sort=asc&order_by=updated_at "noteable_type": "Issue", "noteable_iid": 377, "resolvable": false, - "confidential": false + "confidential": false, + "internal": false }, { "id": 305, @@ -101,7 +102,8 @@ GET /projects/:id/issues/:issue_iid/notes?sort=asc&order_by=updated_at "noteable_type": "Issue", "noteable_iid": 121, "resolvable": false, - "confidential": true + "confidential": true, + "internal": true } ] ``` @@ -145,7 +147,8 @@ Parameters: | `id` | integer or string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding). | | `issue_iid` | integer | yes | The IID of an issue. | | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | -| `confidential` | boolean | no | The confidential flag of a note. Default is false. | +| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0 and renamed to `internal`. The confidential flag of a note. Default is false. | +| `internal` | boolean | no | The internal flag of a note. Overrides `confidential` when both parameters are submitted. Default is false. | | `created_at` | string | no | Date time string, ISO 8601 formatted. It must be after 1970-01-01. Example: `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | ```shell @@ -378,7 +381,8 @@ Parameters: "noteable_type": "MergeRequest", "noteable_iid": 2, "resolvable": false, - "confidential": false + "confidential": false, + "internal": false } ``` @@ -447,7 +451,7 @@ Parameters: curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/merge_requests/7/notes/1602" ``` -## Epics **(ULTIMATE)** +## Epics **(PREMIUM)** ### List all epic notes @@ -506,7 +510,8 @@ Parameters: "expires_at": null, "updated_at": "2013-10-02T07:34:20Z", "created_at": "2013-10-02T07:34:20Z", - "confidential": false + "confidential": false, + "internal": false } ``` @@ -530,7 +535,8 @@ Parameters: | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | | `epic_id` | integer | yes | The ID of an epic | | `id` | integer or string | yes | The ID or [URL-encoded path of the group](index.md#namespaced-path-encoding) | -| `confidential` | boolean | no | The confidential flag of a note. Default is `false`. | +| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0 and is renamed to `internal`. The confidential flag of a note. Default is `false`. | +| `internal` | boolean | no | The internal flag of a note. Overrides `confidential` when both parameters are submitted. Default is `false`. | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/epics/11/notes?body=note" diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 35c6eb4a982..12704f6fc87 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -26,9 +26,12 @@ support [CORS preflight requests](https://developer.mozilla.org/en-US/docs/Web/H - `/oauth/token` - `/oauth/userinfo` -In addition to the headers listed for [simple requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS#simple_requests), -only the `Authorization` header can be used for preflight requests. For example, the `X-Requested-With` header -can't be used for preflight requests. +Only certain headers can be used for preflight requests: + +- The headers listed for [simple requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS#simple_requests). +- The `Authorization` header. + +For example, the `X-Requested-With` header can't be used for preflight requests. ## Supported OAuth 2.0 flows @@ -258,8 +261,8 @@ Check the [RFC spec](https://tools.ietf.org/html/rfc6749#section-4.3) for a detailed flow description. NOTE: -The Resource Owner Password Credentials is disabled for users with [two-factor -authentication](../user/profile/account/two_factor_authentication.md) turned on. +The Resource Owner Password Credentials is disabled for users with +[two-factor authentication](../user/profile/account/two_factor_authentication.md) turned on. These users can access the API using [personal access tokens](../user/profile/personal_access_tokens.md) instead. diff --git a/doc/api/packages.md b/doc/api/packages.md index 9d9c21546cd..3cd93bd09c1 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -6,6 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Packages API **(FREE)** +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349418) support for [GitLab CI/CD job token](../ci/jobs/ci_job_token.md) authentication for the project-level API in GitLab 15.3. + This is the API documentation of [GitLab Packages](../administration/packages/index.md). ## List packages diff --git a/doc/api/packages/conan.md b/doc/api/packages/conan.md index 1590893d006..3ac2eeb40b1 100644 --- a/doc/api/packages/conan.md +++ b/doc/api/packages/conan.md @@ -38,8 +38,8 @@ The examples in this document all use the instance-level prefix. /packages/conan/v1 ``` -When using the instance-level routes, be aware that there is a [naming -restriction](../../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes) +When using the instance-level routes, be aware that there is a +[naming restriction](../../user/packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes) for Conan recipes. ### Project-level diff --git a/doc/api/packages/terraform-modules.md b/doc/api/packages/terraform-modules.md new file mode 100644 index 00000000000..24db7094a3c --- /dev/null +++ b/doc/api/packages/terraform-modules.md @@ -0,0 +1,230 @@ +--- +stage: Package +group: Package +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Terraform Registry API **(FREE)** + +This is the API documentation for [Terraform Modules](../../user/packages/terraform_module_registry/index.md). + +WARNING: +This API is used by the [terraform cli](https://www.terraform.io/) +and is generally not meant for manual consumption. + +For instructions on how to upload and install Maven packages from the GitLab +package registry, see the [Terraform modules registry documentation](../../user/packages/terraform_module_registry/index.md). + +## List available versions for a specific module + +Get a list of available versions for a specific module. + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/versions +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/versions" +``` + +Example response: + +```shell +{ + "modules": [ + { + "versions": [ + { + "version": "1.0.0", + "submodules": [], + "root": { + "dependencies": [], + "providers": [ + { + "name": "local", + "version":"" + } + ] + } + }, + { + "version": "0.9.3", + "submodules": [], + "root": { + "dependencies": [], + "providers": [ + { + "name": "local", + "version":"" + } + ] + } + } + ], + "source": "https://gitlab.example.com/group/hello-world" + } + ] +} +``` + +## Latest version for a specific module + +Get information about the latest version for a given module. + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local" +``` + +Example response: + +```shell +{ + "name": "hellow-world/local", + "provider": "local", + "providers": [ + "local" + ], + "root": { + "dependencies": [] + }, + "source": "https://gitlab.example.com/group/hello-world", + "submodules": [], + "version": "1.0.0", + "versions": [ + "1.0.0" + ] +} +``` + +## Get specific version for a specific module + +Get information about the latest version for a given module. + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/1.0.0 +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0" +``` + +Example response: + +```shell +{ + "name": "hellow-world/local", + "provider": "local", + "providers": [ + "local" + ], + "root": { + "dependencies": [] + }, + "source": "https://gitlab.example.com/group/hello-world", + "submodules": [], + "version": "1.0.0", + "versions": [ + "1.0.0" + ] +} +``` + +## Get URL for downloading latest module version + +Get the download URL for latest module version in `X-Terraform-Get` header + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/download +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/download" +``` + +Example response: + +```shell +HTTP/1.1 204 No Content +Content-Length: 0 +X-Terraform-Get: /api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file?token=&archive=tgz +``` + +Under the hood, this API endpoint redirects to `packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/:module_version/download` + +## Get URL for downloading specific module version + +Get the download URL for a specific module version in `X-Terraform-Get` header + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/:module_version/download +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | +| `module_version` | string | yes | Specific module version to download. | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/download" +``` + +Example response: + +```shell +HTTP/1.1 204 No Content +Content-Length: 0 +X-Terraform-Get: /api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file?token=&archive=tgz +``` + +## Download module + +```plaintext +GET packages/terraform/modules/v1/:module_namespace/:module_name/:module_system/:module_version/file +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `module_namespace` | string | yes | The group to which Terraform module's project belongs. | +| `module_name` | string | yes | The module name. | +| `module_system` | string | yes | The name of the module system or [provider](https://www.terraform.io/registry/providers). | +| `module_version` | string | yes | Specific module version to download. | + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file" +``` + +To write the output to file: + +```shell +curl --header "Private-Token: <personal_access_token>" "https://gitlab.example.com/api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file" --output hello-world-local.tgz +``` diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index 46a4c674560..620b5c2ed0b 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -91,7 +91,12 @@ curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab ### Responses -- `401: Unauthorized` if the user doesn't have access to the token they're requesting the ID or if the token with matching ID doesn't exist. +> `404` HTTP status code [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93650) in GitLab 15.3. + +- `401: Unauthorized` if either: + - The user doesn't have access to the token with the specified ID. + - The token with the specified ID doesn't exist. +- `404: Not Found` if the user is an administrator but the token with the specified ID doesn't exist. ## Revoke a personal access token diff --git a/doc/api/pipeline_schedules.md b/doc/api/pipeline_schedules.md index 625a92f9b89..bbaf38aaaae 100644 --- a/doc/api/pipeline_schedules.md +++ b/doc/api/pipeline_schedules.md @@ -101,6 +101,60 @@ curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/a } ``` +## Get all pipelines triggered by a pipeline schedule + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368566) in GitLab 15.3. + +Get all pipelines triggered by a pipeline schedule in a project. + +```plaintext +GET /projects/:id/pipeline_schedules/:pipeline_schedule_id/pipelines +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +|------------------------|----------------|----------|-------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. | +| `pipeline_schedule_id` | integer | yes | The pipeline schedule ID. | + +Example request: + +```shell +curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/29/pipeline_schedules/13/pipelines" +``` + +Example response: + +```json +[ + { + "id": 47, + "iid": 12, + "project_id": 29, + "status": "pending", + "source": "scheduled", + "ref": "new-pipeline", + "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", + "web_url": "https://example.com/foo/bar/pipelines/47", + "created_at": "2016-08-11T11:28:34.085Z", + "updated_at": "2016-08-11T11:32:35.169Z" + }, + { + "id": 48, + "iid": 13, + "project_id": 29, + "status": "pending", + "source": "scheduled", + "ref": "new-pipeline", + "sha": "eb94b618fb5865b26e80fdd8ae531b7a63ad851a", + "web_url": "https://example.com/foo/bar/pipelines/48", + "created_at": "2016-08-12T10:06:04.561Z", + "updated_at": "2016-08-12T10:09:56.223Z" + } +] +``` + ## Create a new pipeline schedule Create a new pipeline schedule of a project. diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 99009b6913d..2e601f6e24a 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -281,7 +281,7 @@ POST /projects/:id/pipeline |-------------|---------|----------|---------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `ref` | string | yes | The branch or tag to run the pipeline on. | -| `variables` | array | no | An array containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }, {'key': 'TEST', 'value': 'test variable'}]`. If `variable_type` is excluded, it defaults to `env_var`. | +| `variables` | array | no | An [array of hashes](index.md#array-of-hashes) containing the variables available in the pipeline, matching the structure `[{ 'key': 'UPLOAD_TO_S3', 'variable_type': 'file', 'value': 'true' }, {'key': 'TEST', 'value': 'test variable'}]`. If `variable_type` is excluded, it defaults to `env_var`. | ```shell curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/pipeline?ref=main" diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 635cb04e2e8..49a9b68227d 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -15,10 +15,18 @@ See also: Start a new export. -The endpoint also accepts an `upload` parameter. This parameter is a hash. It contains -all the necessary information to upload the exported project to a web server or -to any S3-compatible platform. At the moment we only support binary -data file uploads to the final server. +The endpoint also accepts an `upload` hash parameter. It contains all the necessary information to upload the exported +project to a web server or to any S3-compatible platform. For exports, GitLab: + +- Only supports binary data file uploads to the final server. +- Sends the `Content-Type: application/gzip` header with upload requests. Ensure that your pre-signed URL includes this + as part of the signature. +- Can take some time to complete the project export process. Make sure the upload URL doesn't have a short expiration + time and is available throughout the export process. +- Administrators can modify the maximum export file size. By default, the maximum is unlimited (`0`). To change this, + edit `max_export_size` using either: + - [Application settings API](settings.md#change-application-settings) + - [GitLab UI](../user/admin_area/settings/account_and_limit_settings.md). The `upload[url]` parameter is required if the `upload` parameter is present. @@ -46,15 +54,6 @@ curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitla } ``` -NOTE: -The upload request is sent with `Content-Type: application/gzip` header. Ensure that your pre-signed URL includes this as part of the signature. - -NOTE: -As an administrator, you can modify the maximum export file size. By default, -it is set to `0`, for unlimited. To change this value, edit `max_export_size` -in the [Application settings API](settings.md#change-application-settings) -or the [Admin UI](../user/admin_area/settings/account_and_limit_settings.md). - ## Export status Get the status of export. diff --git a/doc/api/project_templates.md b/doc/api/project_templates.md index 4c0a1890729..7763087e759 100644 --- a/doc/api/project_templates.md +++ b/doc/api/project_templates.md @@ -20,7 +20,7 @@ It deprecates these endpoints, which are scheduled for removal in API version 5. In addition to templates common to the entire instance, project-specific templates are also available from this API endpoint. -Support is also available for [group-level file templates](../user/group/index.md#group-file-templates). **(PREMIUM)** +Support is also available for [group-level file templates](../user/group/manage.md#group-file-templates). **(PREMIUM)** ## Get all templates of a particular type diff --git a/doc/api/projects.md b/doc/api/projects.md index 8a8fe522b63..75eea394a40 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -48,7 +48,6 @@ GET /projects | Attribute | Type | Required | Description | |--------------------------------------------|----------|------------------------|-------------| | `archived` | boolean | **{dotted-circle}** No | Limit by archived status. | -| `build_coverage_regex` | string | **{dotted-circle}** No | Test coverage parsing. (`deprecated`, it is [scheduled to be removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357401)) | | `id_after` | integer | **{dotted-circle}** No | Limit results to projects with IDs greater than the specified ID. | | `id_before` | integer | **{dotted-circle}** No | Limit results to projects with IDs less than the specified ID. | | `imported` | boolean | **{dotted-circle}** No | Limit results to projects which were imported from external systems by current user. | @@ -124,7 +123,7 @@ Example response: "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.example.com/diaspora" - } + } }, { ... @@ -223,12 +222,12 @@ When the user is authenticated and `simple` is not set this returns something li "open_issues_count": 0, "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "public_jobs": true, "build_timeout": 3600, "auto_cancel_pending_pipelines": "enabled", - "build_coverage_regex": null, "ci_config_path": "", "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, @@ -405,6 +404,7 @@ GET /users/:user_id/projects "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_separated_caches": true, "public_jobs": true, "shared_with_groups": [], @@ -513,6 +513,7 @@ GET /users/:user_id/projects "runners_token": "b8547b1dc37721d05889db52fa2f02", "ci_default_git_depth": 0, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_separated_caches": true, "public_jobs": true, "shared_with_groups": [], @@ -920,6 +921,7 @@ GET /projects/:id "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_separated_caches": true, "public_jobs": true, "shared_with_groups": [ @@ -1360,6 +1362,7 @@ Supported attributes: | `ci_config_path` | string | **{dotted-circle}** No | The path to CI configuration file. | | `ci_default_git_depth` | integer | **{dotted-circle}** No | Default number of revisions for [shallow cloning](../ci/pipelines/settings.md#limit-the-number-of-changes-fetched-during-clone). | | `ci_forward_deployment_enabled` | boolean | **{dotted-circle}** No | When a new deployment job starts, [skip older deployment jobs](../ci/pipelines/settings.md#skip-outdated-deployment-jobs) that are still pending | +| `ci_allow_fork_pipelines_to_run_in_parent_project` | boolean | **{dotted-circle}** No | Enable or disable [running pipelines in the parent project for merge requests from forks](../ci/pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project). _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325189) in GitLab 15.3.)_ | | `ci_separated_caches` | boolean | **{dotted-circle}** No | Set whether or not caches should be [separated](../ci/caching/index.md#cache-key-names) by branch protection status. | | `container_expiration_policy_attributes` | hash | **{dotted-circle}** No | Update the image cleanup policy for this project. Accepts: `cadence` (string), `keep_n` (integer), `older_than` (string), `name_regex` (string), `name_regex_delete` (string), `name_regex_keep` (string), `enabled` (boolean). | | `container_registry_access_level` | string | **{dotted-circle}** No | Set visibility of container registry, for this project, to one of `disabled`, `private` or `enabled`. | @@ -1936,6 +1939,7 @@ Example response: "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_separated_caches": true, "public_jobs": true, "shared_with_groups": [], @@ -2062,6 +2066,7 @@ Example response: "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, + "ci_allow_fork_pipelines_to_run_in_parent_project": true, "ci_separated_caches": true, "public_jobs": true, "shared_with_groups": [], @@ -2102,7 +2107,7 @@ This endpoint: is applied if enabled. - From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on [Premium or higher](https://about.gitlab.com/pricing/) tiers, group - administrators can [configure](../user/group/index.md#enable-delayed-project-deletion) + administrators can [configure](../user/group/manage.md#enable-delayed-project-deletion) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after the number of days specified in the [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). @@ -2110,7 +2115,7 @@ This endpoint: WARNING: The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) in GitLab 12.6 was changed to [Immediate deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) -in GitLab 13.2, as discussed in [Enable delayed project deletion](../user/group/index.md#enable-delayed-project-deletion). +in GitLab 13.2, as discussed in [Enable delayed project deletion](../user/group/manage.md#enable-delayed-project-deletion). ```plaintext DELETE /projects/:id @@ -2699,7 +2704,6 @@ Example response: "public_jobs": true, "build_timeout": 3600, "auto_cancel_pending_pipelines": "enabled", - "build_coverage_regex": null, // deprecated, it is scheduled to be removed https://gitlab.com/gitlab-org/gitlab/-/issues/357401 "ci_config_path": null, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 7489d0bdc9e..1332eea26c0 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -384,7 +384,7 @@ POST /projects/:id/releases | `assets:links` | array of hash | no | An array of assets links. | | `assets:links:name`| string | required by: `assets:links` | The name of the link. Link names must be unique within the release. | | `assets:links:url` | string | required by: `assets:links` | The URL of the link. Link URLs must be unique within the release. | -| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). +| `assets:links:filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | `assets:links:link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | `released_at` | datetime | no | The date when the release is/was ready. Defaults to the current time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | @@ -740,4 +740,15 @@ Example response: A release with a `released_at` attribute set to a future date is labeled as an **Upcoming Release** [in the UI](../../user/project/releases/index.md#upcoming-releases). -Additionally, if a [release is requested from the API](#list-releases), for each release with a `release_at` attribute set to a future date, an additional attribute `upcoming_release` (set to true) will be returned as part of the response. +Additionally, if a [release is requested from the API](#list-releases), for each release with a `release_at` attribute set to a future date, an additional attribute `upcoming_release` (set to true) is returned as part of the response. + +## Historical releases + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199429) in GitLab 15.2. + +A release with a `released_at` attribute set to a past date is labeled +as an **Historical release** [in the UI](../../user/project/releases/index.md#historical-releases). + +Additionally, if a [release is requested from the API](#list-releases), for each +release with a `release_at` attribute set to a past date, an additional +attribute `historical_release` (set to true) is returned as part of the response. diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index c239d06ecfd..f9e07991948 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -93,14 +93,14 @@ Create an asset as a link from a Release. POST /projects/:id/releases/:tag_name/assets/links ``` -| Attribute | Type | Required | Description | -| ------------- | -------------- | -------- | ---------------------------------------------------------------------------------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). | -| `tag_name` | string | yes | The tag associated with the Release. | -| `name` | string | yes | The name of the link. Link names must be unique within the release. | -| `url` | string | yes | The URL of the link. Link URLs must be unique within the release. | -| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). | -| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | +| Attribute | Type | Required | Description | +|-------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding). | +| `tag_name` | string | yes | The tag associated with the Release. | +| `name` | string | yes | The name of the link. Link names must be unique in the release. | +| `url` | string | yes | The URL of the link. Link URLs must be unique in the release. | +| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | +| `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | Example request: @@ -141,7 +141,7 @@ PUT /projects/:id/releases/:tag_name/assets/links/:link_id | `link_id` | integer | yes | The ID of the link. | | `name` | string | no | The name of the link. | | `url` | string | no | The URL of the link. | -| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/index.md#permanent-links-to-release-assets). +| `filepath` | string | no | Optional path for a [Direct Asset link](../../user/project/releases/release_fields.md#permanent-links-to-release-assets). | `link_type` | string | no | The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`. | NOTE: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 34b86902271..bf2ead43519 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -125,8 +125,8 @@ Supported attributes: ## Get file archive -> Support for [including Git LFS blobs](../topics/git/lfs/index.md#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5. -> Support for downloading a subfolder was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28827) in GitLab 14.4. +> - Support for [including Git LFS blobs](../topics/git/lfs/index.md#lfs-objects-in-project-archives) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15079) in GitLab 13.5. +> - Support for downloading a subfolder was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/28827) in GitLab 14.4. Get an archive of the repository. This endpoint can be accessed without authentication if the repository is publicly accessible. @@ -319,7 +319,7 @@ Supported attributes: | `date` | datetime | no | The date and time of the release, defaults to the current time. | | `branch` | string | no | The branch to commit the changelog changes to, defaults to the project's default branch. | | `trailer` | string | no | The Git trailer to use for including commits, defaults to `Changelog`. | -| `config_file` | string | no | The path of changelog configuration file in the project's Git repository, defaults to `.gitlab/changelog_config.yml`. | +| `config_file` | string | no | Path to the changelog configuration file in the project's Git repository. Defaults to `.gitlab/changelog_config.yml`. | | `file` | string | no | The file to commit the changes to, defaults to `CHANGELOG.md`. | | `message` | string | no | The commit message to produce when committing the changes, defaults to `Add changelog for version X` where X is the value of the `version` argument. | @@ -342,9 +342,8 @@ tags using these formats: - `vX.Y.Z` - `X.Y.Z` -Where `X.Y.Z` is a version that follows [semantic -versioning](https://semver.org/). For example, consider a project with the -following tags: +Where `X.Y.Z` is a version that follows [semantic versioning](https://semver.org/). +For example, consider a project with the following tags: - v1.0.0-pre1 - v1.0.0 diff --git a/doc/api/resource_iteration_events.md b/doc/api/resource_iteration_events.md index f4463a4746d..37a945b140e 100644 --- a/doc/api/resource_iteration_events.md +++ b/doc/api/resource_iteration_events.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/229463) in GitLab 13.5. > - Moved to GitLab Premium in 13.9. -Resource iteration events keep track of what happens to GitLab [issues](../user/project/issues/). +Resource iteration events keep track of what happens to GitLab [issues](../user/project/issues/index.md). Use them to track which iteration was set, who did it, and when it happened. diff --git a/doc/api/resource_milestone_events.md b/doc/api/resource_milestone_events.md index 75ecd986e4c..448ec6e2c1d 100644 --- a/doc/api/resource_milestone_events.md +++ b/doc/api/resource_milestone_events.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31720) in GitLab 13.1. Resource [milestone](../user/project/milestones/index.md) events keep track of what happens to -GitLab [issues](../user/project/issues/) and [merge requests](../user/project/merge_requests/). +GitLab [issues](../user/project/issues/index.md) and [merge requests](../user/project/merge_requests/index.md). Use them to track which milestone was added or removed, who did it, and when it happened. diff --git a/doc/api/resource_state_events.md b/doc/api/resource_state_events.md index 950d82a92a4..b2e886618d5 100644 --- a/doc/api/resource_state_events.md +++ b/doc/api/resource_state_events.md @@ -8,8 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35210/) in GitLab 13.2. -Resource state events keep track of what happens to GitLab [issues](../user/project/issues/) and -[merge requests](../user/project/merge_requests/). +Resource state events keep track of what happens to GitLab [issues](../user/project/issues/index.md) and +[merge requests](../user/project/merge_requests/index.md). Use them to track which state was set, who did it, and when it happened. diff --git a/doc/api/resource_weight_events.md b/doc/api/resource_weight_events.md index 8f11f0c984b..faa2d543c50 100644 --- a/doc/api/resource_weight_events.md +++ b/doc/api/resource_weight_events.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32542) in GitLab 13.2. -Resource weight events keep track of what happens to GitLab [issues](../user/project/issues/). +Resource weight events keep track of what happens to GitLab [issues](../user/project/issues/index.md). Use them to track which weight was set, who did it, and when it happened. diff --git a/doc/api/settings.md b/doc/api/settings.md index 28a33e4deb4..c736c0df1da 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -403,7 +403,7 @@ listed in the descriptions of the relevant settings. | `performance_bar_enabled` | boolean | no | (Deprecated: Pass `performance_bar_allowed_group_path: nil` instead) Allow enabling the performance bar. | | `personal_access_token_prefix` | string | no | Prefix for all generated personal access tokens. | | `pipeline_limit_per_project_user_sha` | integer | no | Maximum number of pipeline creation requests per minute per user and commit. Disabled by default. | -| `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable PlantUML integration. Default is `false`. | +| `plantuml_enabled` | boolean | no | (**If enabled, requires:** `plantuml_url`) Enable [PlantUML integration](../administration/integration/plantuml.md). Default is `false`. | | `plantuml_url` | string | required by: `plantuml_enabled` | The PlantUML instance URL for integration. | | `polling_interval_multiplier` | decimal | no | Interval multiplier used by endpoints that perform polling. Set to `0` to disable polling. | | `project_export_enabled` | boolean | no | Enable project export. | diff --git a/doc/api/topics.md b/doc/api/topics.md index 78f32fb8fa0..ee88a43ff1c 100644 --- a/doc/api/topics.md +++ b/doc/api/topics.md @@ -20,11 +20,12 @@ GET /topics Supported attributes: -| Attribute | Type | Required | Description | -| ---------- | ------- | ---------------------- | ----------- | -| `page` | integer | **{dotted-circle}** No | Page to retrieve. Defaults to `1`. | -| `per_page` | integer | **{dotted-circle}** No | Number of records to return per page. Defaults to `20`. | -| `search` | string | **{dotted-circle}** No | Search topics against their `name`. | +| Attribute | Type | Required | Description | +| ------------------ | ------- | ---------------------- | ----------- | +| `page` | integer | **{dotted-circle}** No | Page to retrieve. Defaults to `1`. | +| `per_page` | integer | **{dotted-circle}** No | Number of records to return per page. Defaults to `20`. | +| `search` | string | **{dotted-circle}** No | Search topics against their `name`. | +| `without_projects` | boolean | **{dotted-circle}** No | Limit results to topics without assigned projects. | Example request: diff --git a/doc/api/users.md b/doc/api/users.md index 06c0fadbbbf..c30bac3c542 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -39,7 +39,7 @@ GET /users ] ``` -You can also search for users by name, username or public email by using `?search=`. For example. `/users?search=John`. +You can also search for users by name, username, or public email by using `?search=`. For example. `/users?search=John`. In addition, you can lookup users by username: @@ -425,9 +425,10 @@ You can include the user's [custom attributes](custom_attributes.md) in the resp GET /users/:id?with_custom_attributes=true ``` -## User creation +## User creation **(FREE SELF)** -> The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. +> - The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. +> - Ability to create an auditor user was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366404) in GitLab 15.3. Creates a new user. Note only administrators can create new users. Either `password`, `reset_password`, or `force_random_password` @@ -452,7 +453,8 @@ Parameters: | Attribute | Required | Description | | :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `admin` | No | User is an administrator - true or false (default) | +| `admin` | No | User is an administrator. Valid values are `true` or `false`. Defaults to false. +| `auditor` **(PREMIUM)** | No | User is an auditor. Valid values are `true` or `false`. Defaults to false. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366404) in GitLab 15.3. | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | @@ -482,9 +484,10 @@ Parameters: | `view_diffs_file_by_file` | No | Flag indicating the user sees only one file diff per page | | `website_url` | No | Website URL | -## User modification +## User modification **(FREE SELF)** -> The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. +> - The `namespace_id` field in the response was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82045) in GitLab 14.10. +> - Ability to modify an auditor user was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366404) in GitLab 15.3. Modifies an existing user. Only administrators can change attributes of a user. @@ -496,7 +499,8 @@ Parameters: | Attribute | Required | Description | | :----------------------------------- | :------- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `admin` | No | User is an administrator - true or false (default) | +| `admin` | No |User is an administrator. Valid values are `true` or `false`. Defaults to false. +| `auditor` **(PREMIUM)** | No | User is an auditor. Valid values are `true` or `false`. Defaults to false. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366404) in GitLab 15.3.(default) | | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | @@ -531,7 +535,7 @@ Note, at the moment this method does only return a `404` error, even in cases where a `409` (Conflict) would be more appropriate. For example, when renaming the email address to some existing one. -## Delete authentication identity from user +## Delete authentication identity from user **(FREE SELF)** Deletes a user's authentication identity using the provider name associated with that identity. Available only for administrators. @@ -546,7 +550,7 @@ Parameters: | `id` | integer | yes | ID of a user | | `provider` | string | yes | External provider name | -## User deletion +## User deletion **(FREE SELF)** Deletes a user. Available only for administrators. This returns a `204 No Content` status code if the operation was successfully, `404` if the resource was not found or `409` if the user cannot be soft deleted. @@ -753,7 +757,7 @@ PUT /user/status | Attribute | Type | Required | Description | | -------------------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `emoji` | string | no | Name of the emoji to use as status. If omitted `speech_balloon` is used. Emoji name can be one of the specified names in the [Gemojione index](https://github.com/bonusly/gemojione/blob/master/config/index.json). | -| `message` | string | no | Message to set as a status. It can also contain emoji codes. | +| `message` | string | no | Message to set as a status. It can also contain emoji codes. Cannot exceed 100 characters. | | `clear_status_after` | string | no | Automatically clean up the status after a given time interval, allowed values: `30_minutes`, `3_hours`, `8_hours`, `1_day`, `3_days`, `7_days`, `30_days` When both parameters `emoji` and `message` are empty, the status is cleared. When the `clear_status_after` parameter is missing from the request, the previously set value for `"clear_status_after` is cleared. @@ -1069,7 +1073,7 @@ error occurs a `400 Bad Request` is returned with a message explaining the error } ``` -## Add SSH key for user +## Add SSH key for user **(FREE SELF)** Create new key owned by specified user. Available only for administrator. @@ -1104,7 +1108,7 @@ Parameters: |-----------|---------|----------|-------------| | `key_id` | integer | yes | SSH key ID | -## Delete SSH key for given user +## Delete SSH key for given user **(FREE SELF)** Deletes key owned by a specified user. Available only for administrator. @@ -1220,7 +1224,7 @@ Parameters: curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/user/gpg_keys/1" ``` -Returns `204 No Content` on success, or `404 Not found` if the key cannot be found. +Returns `204 No Content` on success or `404 Not Found` if the key cannot be found. ## List all GPG keys for given user @@ -1282,7 +1286,7 @@ Example response: } ``` -## Add a GPG key for a given user +## Add a GPG key for a given user **(FREE SELF)** Create new GPG key owned by the specified user. Available only for administrator. @@ -1314,7 +1318,7 @@ Example response: ] ``` -## Delete a GPG key for a given user +## Delete a GPG key for a given user **(FREE SELF)** Delete a GPG key owned by a specified user. Available only for administrator. @@ -1364,7 +1368,7 @@ Parameters: - **none** -## List emails for user +## List emails for user **(FREE SELF)** Get a list of a specified user's emails. Available only for administrator @@ -1439,7 +1443,7 @@ error occurs a `400 Bad Request` is returned with a message explaining the error } ``` -## Add email for user +## Add email for user **(FREE SELF)** Create new email owned by specified user. Available only for administrator @@ -1470,7 +1474,7 @@ Parameters: |------------|---------|----------|-------------| | `email_id` | integer | yes | Email ID | -## Delete email for given user +## Delete email for given user **(FREE SELF)** Deletes email owned by a specified user. Available only for administrator. @@ -1485,7 +1489,7 @@ Parameters: | `id` | integer | yes | ID of specified user | | `email_id` | integer | yes | Email ID | -## Block user +## Block user **(FREE SELF)** Blocks the specified user. Available only for administrator. @@ -1507,7 +1511,7 @@ Returns: - A user that is blocked through LDAP. - An internal user. -## Unblock user +## Unblock user **(FREE SELF)** Unblocks the specified user. Available only for administrator. @@ -1524,7 +1528,7 @@ Parameters: Returns `201 OK` on success, `404 User Not Found` is user cannot be found or `403 Forbidden` when trying to unblock a user blocked by LDAP synchronization. -## Deactivate user +## Deactivate user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. @@ -1549,7 +1553,7 @@ Returns: - That has any activity in past 90 days. These users cannot be deactivated. - That is internal. -## Activate user +## Activate user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22257) in GitLab 12.4. @@ -1571,7 +1575,7 @@ Returns: - `404 User Not Found` if the user cannot be found. - `403 Forbidden` if the user cannot be activated because they are blocked by an administrator or by LDAP synchronization. -## Ban user +## Ban user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327354) in GitLab 14.3. @@ -1591,7 +1595,7 @@ Returns: - `404 User Not Found` if user cannot be found. - `403 Forbidden` when trying to ban a user that is not active. -## Unban user +## Unban user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327354) in GitLab 14.3. @@ -1615,7 +1619,7 @@ Returns: Please refer to the [Events API documentation](events.md#get-user-contribution-events) -## Get all impersonation tokens of a user +## Get all impersonation tokens of a user **(FREE SELF)** Requires administrator access. @@ -1670,7 +1674,7 @@ Example response: ] ``` -## Approve user +## Approve user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263107) in GitLab 13.7. @@ -1711,7 +1715,7 @@ Example Responses: { "message": "The user you are trying to approve is not pending approval" } ``` -## Reject user +## Reject user **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339925) in GitLab 14.3. @@ -1750,7 +1754,7 @@ Example Responses: { "message": "User does not have a pending request" } ``` -## Get an impersonation token of a user +## Get an impersonation token of a user **(FREE SELF)** > Requires administrators permissions. @@ -1789,7 +1793,7 @@ Example response: } ``` -## Create an impersonation token +## Create an impersonation token **(FREE SELF)** Requires administrator access. Token values are returned once. Make sure you save it because you can't access it again. @@ -1834,7 +1838,7 @@ Example response: } ``` -## Revoke an impersonation token +## Revoke an impersonation token **(FREE SELF)** Requires administrator access. @@ -1898,7 +1902,7 @@ Example response: } ``` -## Get user activities +## Get user activities **(FREE SELF)** Pre-requisite: @@ -1955,7 +1959,7 @@ Example response: `last_activity_at` is deprecated. Use `last_activity_on` instead. -## User memberships +## User memberships **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20532) in GitLab 12.8. @@ -1964,7 +1968,7 @@ Pre-requisite: - You must be an administrator. Lists all projects and groups a user is a member of. -It returns the `source_id`, `source_name`, `source_type` and `access_level` of a membership. +It returns the `source_id`, `source_name`, `source_type`, and `access_level` of a membership. Source can be of type `Namespace` (representing a group) or `Project`. The response represents only direct memberships. Inherited memberships, for example in subgroups, are not included. Access levels are represented by an integer value. For more details, read about the meaning of [access level values](access_requests.md#valid-access-levels). @@ -2009,7 +2013,7 @@ Example response: ] ``` -## Disable two factor authentication +## Disable two factor authentication **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/295260) in GitLab 15.2. diff --git a/doc/api/version.md b/doc/api/version.md index 80269bc5697..7d072e23410 100644 --- a/doc/api/version.md +++ b/doc/api/version.md @@ -6,6 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Version API **(FREE)** +NOTE: +We recommend you use the [Metadata API](metadata.md) instead of the Version API. +It contains additional information and is aligned with the GraphQL metadata endpoint. + Retrieve version information for this GitLab instance. Responds `200 OK` for authenticated users. diff --git a/doc/api/vulnerabilities.md b/doc/api/vulnerabilities.md index 18d97e30643..66d0579bacb 100644 --- a/doc/api/vulnerabilities.md +++ b/doc/api/vulnerabilities.md @@ -281,7 +281,7 @@ with the GraphQL API. ### GraphQL - Single vulnerability -Use [`Query.vulnerability`](graphql/reference/#queryvulnerability). +Use [`Query.vulnerability`](graphql/reference/index.md#queryvulnerability). ```graphql { @@ -337,7 +337,7 @@ Example response: ### GraphQL - Confirm vulnerability -Use [`Mutation.vulnerabilityConfirm`](graphql/reference/#mutationvulnerabilityconfirm). +Use [`Mutation.vulnerabilityConfirm`](graphql/reference/index.md#mutationvulnerabilityconfirm). ```graphql mutation { @@ -367,7 +367,7 @@ Example response: ### GraphQL - Resolve vulnerability -Use [`Mutation.vulnerabilityResolve`](graphql/reference/#mutationvulnerabilityresolve). +Use [`Mutation.vulnerabilityResolve`](graphql/reference/index.md#mutationvulnerabilityresolve). ```graphql mutation { @@ -397,7 +397,7 @@ Example response: ### GraphQL - Dismiss vulnerability -Use [`Mutation.vulnerabilityDismiss`](graphql/reference/#mutationvulnerabilitydismiss). +Use [`Mutation.vulnerabilityDismiss`](graphql/reference/index.md#mutationvulnerabilitydismiss). ```graphql mutation { @@ -427,7 +427,7 @@ Example response: ### GraphQL - Revert vulnerability to detected state -Use [`Mutation.vulnerabilityRevertToDetected`](graphql/reference/#mutationvulnerabilityreverttodetected). +Use [`Mutation.vulnerabilityRevertToDetected`](graphql/reference/index.md#mutationvulnerabilityreverttodetected). ```graphql mutation { diff --git a/doc/api/vulnerability_findings.md b/doc/api/vulnerability_findings.md index 20bbe66549d..46479009d7d 100644 --- a/doc/api/vulnerability_findings.md +++ b/doc/api/vulnerability_findings.md @@ -148,7 +148,7 @@ with the GraphQL API. ### GraphQL - Project vulnerabilities -Use [`Project.vulnerabilities`](graphql/reference/#projectvulnerabilities). +Use [`Project.vulnerabilities`](graphql/reference/index.md#projectvulnerabilities). ```graphql { diff --git a/doc/architecture/blueprints/ci_data_decay/index.md b/doc/architecture/blueprints/ci_data_decay/index.md index 8808a526df0..7c0bdf299db 100644 --- a/doc/architecture/blueprints/ci_data_decay/index.md +++ b/doc/architecture/blueprints/ci_data_decay/index.md @@ -48,8 +48,8 @@ PostgreSQL database running on GitLab.com. This volume contributes to significant performance problems, development challenges and is often related to production incidents. -We also expect a [significant growth in the number of builds executed on -GitLab.com](../ci_scale/index.md) in the upcoming years. +We also expect a [significant growth in the number of builds executed on GitLab.com](../ci_scale/index.md) +in the upcoming years. ## Opportunity @@ -61,8 +61,8 @@ pipelines that are older than a few months might help us to move this data out of the primary database, to a different storage, that is more performant and cost effective. -It is already possible to prevent processing builds [that have been -archived](../../../user/admin_area/settings/continuous_integration.md#archive-jobs). +It is already possible to prevent processing builds +[that have been archived](../../../user/admin_area/settings/continuous_integration.md#archive-jobs). When a build gets archived it will not be possible to retry it, but we still do keep all the processing metadata in the database, and it consumes resources that are scarce in the primary database. diff --git a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md index 60b20c50696..868dae4fc6c 100644 --- a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md +++ b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md @@ -306,9 +306,19 @@ We also need to build a proof of concept for removing data on the PostgreSQL side (using foreign keys with `ON DELETE CASCADE`) and removing data through Rails associations, as this might be an important area of uncertainty. -We need to [better understand](https://gitlab.com/gitlab-org/gitlab/-/issues/360148) -how unique constraints we are currently using will perform when using the -partitioned schema. +We [learned](https://gitlab.com/gitlab-org/gitlab/-/issues/360148) that `PostgreSQL` +does not allow to create a single index (unique or otherwise) across all partitions of a table. + +One solution to solve this problem is to embed the partitioning key inside the uniqueness constraint. + +This might mean prepending the partition ID in a hexadecimal format before the token itself and storing +the concatenated string in a database. To do that we would need to reserve an appropriate number of +leading bytes in a token to accommodate for the maximum number of partitions we may have in the future. +It seems that reserving four characters, what would translate into 16-bits number in base-16, +might be sufficient. The maximum number we can encode this way would be FFFF, what is 65535 in decimal. + +This would provide a unique constraint per-partition which +is sufficient for global uniqueness. We have also designed a query analyzer that makes it possible to detect direct usage of zero partitions, legacy tables that have been attached as first diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index 1c3aee2f860..5822ae2b5ed 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -89,8 +89,8 @@ environment. We also expect a significant, exponential growth in the upcoming years. -One of the forecasts done using [Facebook's -Prophet](https://facebook.github.io/prophet/) shows that in the first half of +One of the forecasts done using [Facebook's Prophet](https://facebook.github.io/prophet/) +shows that in the first half of 2024 we expect seeing 20M builds created on GitLab.com each day. In comparison to around 2M we see created today, this is 10x growth our product might need to sustain in upcoming years. @@ -115,17 +115,14 @@ of the CI/CD Apdex score, and sometimes even causes a significant performance degradation in the production environment. There are multiple other strategies that can improve performance and -reliability. We can use [Redis -queuing](https://gitlab.com/gitlab-org/gitlab/-/issues/322972), or [a separate -table that will accelerate SQL queries used to build -queues](https://gitlab.com/gitlab-org/gitlab/-/issues/322766) and we want to -explore them. - -**Status**: As of October 2021 the new architecture [has been implemented on -GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/5909#note_680407908). -The following epic tracks making it generally available: [Make the new pending -builds architecture generally available]( -https://gitlab.com/groups/gitlab-org/-/epics/6954). +reliability. We can use [Redis queuing](https://gitlab.com/gitlab-org/gitlab/-/issues/322972), or +[a separate table that will accelerate SQL queries used to build queues](https://gitlab.com/gitlab-org/gitlab/-/issues/322766) +and we want to explore them. + +**Status**: As of October 2021 the new architecture +[has been implemented on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/5909#note_680407908). +The following epic tracks making it generally available: +[Make the new pending builds architecture generally available](https://gitlab.com/groups/gitlab-org/-/epics/6954). ### Moving big amounts of data is challenging @@ -135,8 +132,7 @@ stores more than 600 gigabytes of data, and `ci_builds.yaml_variables` more than 300 gigabytes (as of February 2021). It is a lot of data that needs to be reliably moved to a different place. -Unfortunately, right now, our [background -migrations](https://docs.gitlab.com/ee/development/background_migrations.html) +Unfortunately, right now, our [background migrations](../../../development/database/background_migrations.md) are not reliable enough to migrate this amount of data at scale. We need to build mechanisms that will give us confidence in moving this data between columns, tables, partitions or database shards. diff --git a/doc/architecture/blueprints/cloud_native_build_logs/index.md b/doc/architecture/blueprints/cloud_native_build_logs/index.md index 3aba10fc758..0c941e332cb 100644 --- a/doc/architecture/blueprints/cloud_native_build_logs/index.md +++ b/doc/architecture/blueprints/cloud_native_build_logs/index.md @@ -12,8 +12,8 @@ Cloud native and the adoption of Kubernetes has been recognised by GitLab to be one of the top two biggest tailwinds that are helping us grow faster as a company behind the project. -This effort is described in a more details [in the infrastructure team -handbook](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). +This effort is described in a more details +[in the infrastructure team handbook](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). ## Traditional build logs @@ -88,9 +88,8 @@ even tried to replace NFS with Since that time it has become apparent that the cost of operations and maintenance of a NFS cluster is significant and that if we ever decide to -migrate to Kubernetes [we need to decouple GitLab from a shared local storage -and -NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). +migrate to Kubernetes +[we need to decouple GitLab from a shared local storage and NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). 1. NFS might be a single point of failure 1. NFS can only be reliably scaled vertically @@ -112,12 +111,10 @@ of complexity, maintenance cost and enormous, negative impact on availability. 1. ✓ Rollout the feature into production environment incrementally The work needed to make the new architecture production ready and enabled on -GitLab.com had been tracked in [Cloud Native Build Logs on -GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/4275) epic. +GitLab.com had been tracked in [Cloud Native Build Logs on GitLab.com](https://gitlab.com/groups/gitlab-org/-/epics/4275) epic. -Enabling this feature on GitLab.com is a subtask of [making the new -architecture generally -available](https://gitlab.com/groups/gitlab-org/-/epics/3791) for everyone. +Enabling this feature on GitLab.com is a subtask of +[making the new architecture generally available](https://gitlab.com/groups/gitlab-org/-/epics/3791) for everyone. ## Status diff --git a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md index e545e8844ec..89c3a4cd6b4 100644 --- a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md +++ b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md @@ -17,8 +17,8 @@ Cloud Native and the adoption of Kubernetes has been recognised by GitLab to be one of the top two biggest tailwinds that are helping us grow faster as a company behind the project. -This effort is described in more detail [in the infrastructure team handbook -page](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). +This effort is described in more detail +[in the infrastructure team handbook page](https://about.gitlab.com/handbook/engineering/infrastructure/production/kubernetes/gitlab-com/). GitLab Pages is tightly coupled with NFS and in order to unblock Kubernetes migration a significant change to GitLab Pages' architecture is required. This @@ -55,9 +55,8 @@ even tried to replace NFS with Since that time it has become apparent that the cost of operations and maintenance of a NFS cluster is significant and that if we ever decide to -migrate to Kubernetes [we need to decouple GitLab from a shared local storage -and -NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). +migrate to Kubernetes +[we need to decouple GitLab from a shared local storage and NFS](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/426#note_375646396). 1. NFS might be a single point of failure 1. NFS can only be reliably scaled vertically @@ -84,8 +83,8 @@ graph TD C -- Serves static content --> E(Visitors) ``` -This new architecture has been briefly described in [the blog -post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/) +This new architecture has been briefly described in +[the blog post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/) too. ## Iterations diff --git a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md index 53ea56b8724..df8686ed0aa 100644 --- a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md +++ b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md @@ -131,7 +131,7 @@ epic. The initial iteration will provide a framework to house features under `Namespaces`. Stage groups will eventually need to migrate their own features and functionality over to `Namespaces`. This may impact these features in unexpected ways. Therefore, to minimize UX debt and maintain product consistency, stage groups will have to consider a number of factors when migrating their features over to `Namespaces`: -1. **Conceptual model**: What are the current and future state conceptual models of these features ([see object modeling for designers](https://hpadkisson.medium.com/object-modeling-for-designers-an-introduction-7871bdcf8baf))? These should be documented in Pajamas (example: [merge requests](https://design.gitlab.com/objects/merge-request)). +1. **Conceptual model**: What are the current and future state conceptual models of these features ([see object modeling for designers](https://hpadkisson.medium.com/object-modeling-for-designers-an-introduction-7871bdcf8baf))? These should be documented in Pajamas (example: [merge requests](https://design.gitlab.com/objects/merge-request/)). 1. **Merge conflicts**: What inconsistencies are there across project, group, and administrator levels? How might these be addressed? For an example of how we rationalized this for labels, please see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338820). 1. **Inheritance & information flow**: How is information inherited across our container hierarchy currently? How might this be impacted if complying with the new [inheritance behavior](https://gitlab.com/gitlab-org/gitlab/-/issues/343316) framework? 1. **Settings**: Where can settings for this feature be found currently? How will these be impacted by `Namespaces`? diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index 5105f2e08e2..9d40593d7ce 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -78,7 +78,7 @@ The single entrypoint for the registry is the [HTTP API](https://gitlab.com/gitl | Operation | UI | Background | Observations | | ------------------------------------------------------------ | ------------------ | ------------------------ | ------------------------------------------------------------ | | [Check API version](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#api-version-check) | **{check-circle}** Yes | **{check-circle}** Yes | Used globally to ensure that the registry supports the Docker Distribution V2 API, as well as for identifying whether GitLab Rails is talking to the GitLab Container Registry or a third-party one (used to toggle features only available in the former). | -| [List repository tags](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#listing-image-tags) | **{check-circle}** Yes | **{check-circle}** Yes | Used to list and show tags in the UI. Used to list tags in the background for [cleanup policies](../../../user/packages/container_registry/reduce_container_registry_storage.md#cleanup-policy) and [Geo replication](../../../administration/geo/replication/docker_registry.md). | +| [List repository tags](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#listing-image-tags) | **{check-circle}** Yes | **{check-circle}** Yes | Used to list and show tags in the UI. Used to list tags in the background for [cleanup policies](../../../user/packages/container_registry/reduce_container_registry_storage.md#cleanup-policy) and [Geo replication](../../../administration/geo/replication/container_registry.md). | | [Check if manifest exists](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#existing-manifests) | **{check-circle}** Yes | **{dotted-circle}** No | Used to get the digest of a manifest by tag. This is then used to pull the manifest and show the tag details in the UI. | | [Pull manifest](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#pulling-an-image-manifest) | **{check-circle}** Yes | **{dotted-circle}** No | Used to show the image size and the manifest digest in the tag details UI. | | [Pull blob](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/spec/api.md#pulling-a-layer) | **{check-circle}** Yes | **{dotted-circle}** No | Used to show the configuration digest and the creation date in the tag details UI. | @@ -148,7 +148,7 @@ The interaction between the registry and its clients, including GitLab Rails and Following the GitLab [Go standards and style guidelines](../../../development/go_guide), no ORM is used to manage the database, only the [`database/sql`](https://pkg.go.dev/database/sql) package from the Go standard library, a PostgreSQL driver ([`lib/pq`](https://pkg.go.dev/github.com/lib/pq?tab=doc)) and raw SQL queries, over a TCP connection pool. -The design and development of the registry database adhere to the GitLab [database guidelines](../../../development/database/). Being a Go application, the required tooling to support the database will have to be developed, such as for running database migrations. +The design and development of the registry database adhere to the GitLab [database guidelines](../../../development/database/index.md). Being a Go application, the required tooling to support the database will have to be developed, such as for running database migrations. Running *online* and [*post deployment*](../../../development/database/post_deployment_migrations.md) migrations is already supported by the registry CLI, as described in the [documentation](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-migrations.md). @@ -160,7 +160,7 @@ Although blobs are shared across repositories, manifest and tag metadata are sco #### GitLab.com -Due to scale, performance and isolation concerns, for GitLab.com the registry database will be on a separate dedicated PostgreSQL cluster. Please see [#93](https://gitlab.com/gitlab-org/container-registry/-/issues/93) and [GitLab-com/gl-infra/infrastructure#10109](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/10109) for additional context. +Due to scale, performance and isolation concerns, for GitLab.com the registry database will be on a separate dedicated PostgreSQL cluster. Please see [#93](https://gitlab.com/gitlab-org/container-registry/-/issues/93) and [GitLab-com/gl-infra/reliability#10109](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/10109) for additional context. The diagram below illustrates the architecture of the database cluster: @@ -272,7 +272,7 @@ The expected registry behavior will be covered with integration tests, using a p Apart from unusual network and systems conditions, problematic migrations and data failures can also affect the database availability and, as a consequence, the registry availability. -Database migrations will adhere to the same [development best practices](../../../development/database/) used for GitLab Rails, except Rails-specific methods and tools, as the registry is a Go application with no ORM, and migrations are therefore expressed with raw SQL statements. Regardless, all changes will require a review and approval from the Database team. +Database migrations will adhere to the same [development best practices](../../../development/database/index.md) used for GitLab Rails, except Rails-specific methods and tools, as the registry is a Go application with no ORM, and migrations are therefore expressed with raw SQL statements. Regardless, all changes will require a review and approval from the Database team. Database migrations will be idempotent, with guard clauses used whenever necessary. It is also intended to guarantee that they are forward compatible. For a clustered environment, a node running registry version `N` should not cause any issues when the database schema is from version `N`+1. @@ -280,9 +280,9 @@ Database migrations will be idempotent, with guard clauses used whenever necessa Adding one more component to the system makes it even more critical to guarantee that we have proper observability over the registry's behavior and its dependencies. It should be guaranteed that we have all the necessary tools in place before rolling out the new registry backed by the metadata database. -For this purpose, [error reporting with Sentry](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/11297), [improved structured logging](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/10933), and [improved HTTP metrics](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/10935) were already implemented and released. At the time of writing, the GitLab.com rollout is in progress. +For this purpose, [error reporting with Sentry](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/11297), [improved structured logging](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/10933), and [improved HTTP metrics](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/10935) were already implemented and released. At the time of writing, the GitLab.com rollout is in progress. -Additionally, the Prometheus metrics will be augmented with [details on the database connection pool](https://gitlab.com/gitlab-org/container-registry/-/issues/238). These will be added to the registry Grafana dashboards, joining the existing HTTP API, deployment, and storage metrics. The database cluster for the registry [will also have metrics and alerts](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/11447). +Additionally, the Prometheus metrics will be augmented with [details on the database connection pool](https://gitlab.com/gitlab-org/container-registry/-/issues/238). These will be added to the registry Grafana dashboards, joining the existing HTTP API, deployment, and storage metrics. The database cluster for the registry [will also have metrics and alerts](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/11447). Together, these resources should provide an adequate level of insight into the registry's performance and behavior. @@ -343,7 +343,7 @@ A more detailed list of all tasks, as well as periodic progress updates can be f - [Proposal for continuous, on-demand online garbage collection](https://gitlab.com/gitlab-org/container-registry/-/issues/199) - [Gradual migration proposal for the GitLab.com container registry](https://gitlab.com/gitlab-org/container-registry/-/issues/191) - [Create a self-serve registry deployment](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/316) -- [Database cluster for container registry](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/11154) +- [Database cluster for container registry](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/11154) ## Who diff --git a/doc/architecture/blueprints/database_scaling/size-limits.md b/doc/architecture/blueprints/database_scaling/size-limits.md index 375c82f8833..284f6402d3c 100644 --- a/doc/architecture/blueprints/database_scaling/size-limits.md +++ b/doc/architecture/blueprints/database_scaling/size-limits.md @@ -136,7 +136,7 @@ As such, the target size of a physical table after refactoring depends on the si There is no standard solution to reduce table sizes - there are many! 1. **Retention**: Delete unnecessary data, for example expire old and unneeded records. -1. **Remove STI**: We still use [single-table inheritance](../../../development/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables. +1. **Remove STI**: We still use [single-table inheritance](../../../development/database/single_table_inheritance.md) in a few places, which is considered an anti-pattern. Redesigning this, we can split data into multiple tables. 1. **Index optimization**: Drop unnecessary indexes and consolidate overlapping indexes if possible. 1. **Optimise data types**: Review data type decisions and optimise data types where possible (example: use integer instead of text for an enum column) 1. **Partitioning**: Apply a partitioning scheme if there is a common access dimension. @@ -167,7 +167,7 @@ The [epic for `~group::database`](https://gitlab.com/groups/gitlab-org/-/epics/6 <!-- vale gitlab.Spelling = NO --> -Identifying solutions for offending tables is driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/database/) and respective stage groups. +Identifying solutions for offending tables is driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/) and respective stage groups. | Role | Who |------------------------------|-------------------------| diff --git a/doc/architecture/blueprints/database_testing/index.md b/doc/architecture/blueprints/database_testing/index.md index 30ebd06c81f..5bc9528d568 100644 --- a/doc/architecture/blueprints/database_testing/index.md +++ b/doc/architecture/blueprints/database_testing/index.md @@ -127,7 +127,7 @@ An alternative approach we have discussed and abandoned is to "scrub" and anonym <!-- vale gitlab.Spelling = NO --> -This effort is owned and driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/database/) with support from the [GitLab.com Reliability Datastores](https://about.gitlab.com/handbook/engineering/infrastructure/team/reliability/) team. +This effort is owned and driven by the [GitLab Database Team](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/) with support from the [GitLab.com Reliability Datastores](https://about.gitlab.com/handbook/engineering/infrastructure/team/reliability/) team. | Role | Who |------------------------------|-------------------------| diff --git a/doc/architecture/blueprints/feature_flags_development/index.md b/doc/architecture/blueprints/feature_flags_development/index.md index 94d52ea3474..eaca7da6bd7 100644 --- a/doc/architecture/blueprints/feature_flags_development/index.md +++ b/doc/architecture/blueprints/feature_flags_development/index.md @@ -115,9 +115,9 @@ These are reason why these changes are needed: ## Iterations -This work is being done as part of dedicated epic: [Improve internal usage of -Feature Flags](https://gitlab.com/groups/gitlab-org/-/epics/3551). This epic -describes a meta reasons for making these changes. +This work is being done as part of dedicated epic: +[Improve internal usage of Feature Flags](https://gitlab.com/groups/gitlab-org/-/epics/3551). +This epic describes a meta reasons for making these changes. ## Who diff --git a/doc/architecture/blueprints/graphql_api/index.md b/doc/architecture/blueprints/graphql_api/index.md index b3ba1ad1960..eb045de491e 100644 --- a/doc/architecture/blueprints/graphql_api/index.md +++ b/doc/architecture/blueprints/graphql_api/index.md @@ -20,8 +20,7 @@ GraphQL development and helped to surface the need of improving tooling we use to extend the new API. This document describes the work that is needed to build a stable foundation that -will support our development efforts and a large-scale usage of the [GraphQL -API](https://docs.gitlab.com/ee/api/graphql/index.html). +will support our development efforts and a large-scale usage of the [GraphQL API](../../../api/graphql/index.md). ## Summary @@ -45,12 +44,12 @@ It is an opportunity to learn from our experience in evolving the REST API, for the scale, and to apply this knowledge onto the GraphQL development efforts. We can do that by building query-to-feature correlation mechanisms, adding scalable state synchronization support and aligning GraphQL with other -architectural initiatives being executed in parallel, like [the support for -direct uploads](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). +architectural initiatives being executed in parallel, like +[the support for direct uploads](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). GraphQL should be secure by default. We can avoid common security mistakes by -building mechanisms that will help us to enforce [OWASP GraphQL -recommendations](https://cheatsheetseries.owasp.org/cheatsheets/GraphQL_Cheat_Sheet.html) +building mechanisms that will help us to enforce +[OWASP GraphQL recommendations](https://cheatsheetseries.owasp.org/cheatsheets/GraphQL_Cheat_Sheet.html) that are relevant to us. Understanding what are the needs of the wider community will also allow us to diff --git a/doc/architecture/blueprints/object_storage/index.md b/doc/architecture/blueprints/object_storage/index.md index 3420caa325f..b70339c8b8d 100644 --- a/doc/architecture/blueprints/object_storage/index.md +++ b/doc/architecture/blueprints/object_storage/index.md @@ -31,9 +31,9 @@ underlying implementation for shared, distributed, highly-available (HA) file storage. Over time, we have built support for object storage across the -application, solving specific problems in a [multitude of -iterations](https://about.gitlab.com/company/team/structure/working-groups/object-storage/#company-efforts-on-uploads). This -has led to increased complexity across the board, from development +application, solving specific problems in a +[multitude of iterations](https://about.gitlab.com/company/team/structure/working-groups/object-storage/#company-efforts-on-uploads). +This has led to increased complexity across the board, from development (new features and bug fixes) to installation: - New GitLab installations require the creation and configuration of @@ -67,10 +67,8 @@ has led to increased complexity across the board, from development The following is a brief description of the main directions we can take to remove the pain points affecting our object storage implementation. -This is also available as [a YouTube -video](https://youtu.be/X9V_w8hsM8E) recorded for the [Object Storage -Working -Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/). +This is also available as [a YouTube video](https://youtu.be/X9V_w8hsM8E) recorded for the +[Object Storage Working Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/). ### Simplify GitLab architecture by shipping MinIO @@ -80,8 +78,8 @@ local storage and object storage. With local storage, there is the assumption of a shared storage between components. This can be achieved by having a single box -installation, without HA, or with a NFS, which [we no longer -recommend](../../../administration/nfs.md). +installation, without HA, or with a NFS, which +[we no longer recommend](../../../administration/nfs.md). We have a testing gap on object storage. It also requires Workhorse and MinIO, which are not present in our pipelines, so too much is @@ -136,8 +134,8 @@ access to new features without infrastructure chores. Our implementation is built on top of a 3rd-party framework where every object storage client is a 3rd-party library. Unfortunately some -of them are unmaintained. [We have customers who cannot push 5GB Git -LFS objects](https://gitlab.com/gitlab-org/gitlab/-/issues/216442), +of them are unmaintained. +[We have customers who cannot push 5GB Git LFS objects](https://gitlab.com/gitlab-org/gitlab/-/issues/216442), but with such a vital feature implemented in 3rd-party libraries we are slowed down in fixing it, and we also rely on external maintainers to merge and release fixes. @@ -147,8 +145,7 @@ Before the introduction of direct upload, using the library, _"a gem that provides a simple and extremely flexible way to upload files from Ruby applications."_, was the boring solution. However this is no longer our use-case, as we upload files from -Workhorse, and we had to [patch CarrierWave's -internals](https://gitlab.com/gitlab-org/gitlab/-/issues/285597#note_452696638) +Workhorse, and we had to [patch CarrierWave's internals](https://gitlab.com/gitlab-org/gitlab/-/issues/285597#note_452696638) to support direct upload. A brief proposal covering CarrierWave removal and a new streamlined @@ -217,7 +214,7 @@ Proposal: DRIs: -The DRI for this blueprint is the [Object Storage Working -Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/). +The DRI for this blueprint is the +[Object Storage Working Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/). <!-- vale gitlab.Spelling = YES --> diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md index c4bd8433ab3..494aaa6a641 100644 --- a/doc/architecture/blueprints/runner_scaling/index.md +++ b/doc/architecture/blueprints/runner_scaling/index.md @@ -33,8 +33,8 @@ This design choice was crucial for the GitLab Runner success. Since that time the auto-scaling feature has been used by many users and customers and enabled rapid growth of CI/CD adoption on GitLab.com. -We can not, however, continue using Docker Machine. Work on that project [was -paused in July 2018](https://github.com/docker/machine/issues/4537) and there +We can not, however, continue using Docker Machine. Work on that project +[was paused in July 2018](https://github.com/docker/machine/issues/4537) and there was no development made since that time (except for some highly important security fixes). In 2018, after Docker Machine entered the "maintenance mode", we decided to create [our own fork](https://gitlab.com/gitlab-org/ci-cd/docker-machine) @@ -76,8 +76,8 @@ mechanism with a reliable and flexible mechanism. We might be unable to build a drop-in replacement for Docker Machine, as there are presumably many reasons why it has been deprecated. It is very difficult to maintain compatibility with so many cloud providers, and it seems that Docker Machine has been deprecated -in favor of Docker Desktop, which is not a viable replacement for us. [This -issue](https://github.com/docker/roadmap/issues/245) contains a discussion +in favor of Docker Desktop, which is not a viable replacement for us. +[This issue](https://github.com/docker/roadmap/issues/245) contains a discussion about how people are using Docker Machine right now, and it seems that GitLab CI is one of the most frequent reasons for people to keep using Docker Machine. diff --git a/doc/ci/cloud_services/aws/index.md b/doc/ci/cloud_services/aws/index.md index dc8dfd95899..2e1abf83a11 100644 --- a/doc/ci/cloud_services/aws/index.md +++ b/doc/ci/cloud_services/aws/index.md @@ -62,16 +62,14 @@ After you configure the OIDC and role, the GitLab CI/CD job can retrieve a tempo assume role: script: - > - STS=($(aws sts assume-role-with-web-identity + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" + $(aws sts assume-role-with-web-identity --role-arn ${ROLE_ARN} --role-session-name "GitLabRunner-${CI_PROJECT_ID}-${CI_PIPELINE_ID}" --web-identity-token $CI_JOB_JWT_V2 --duration-seconds 3600 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' --output text)) - - export AWS_ACCESS_KEY_ID="${STS[0]}" - - export AWS_SECRET_ACCESS_KEY="${STS[1]}" - - export AWS_SESSION_TOKEN="${STS[2]}" - aws sts get-caller-identity ``` diff --git a/doc/ci/cloud_services/google_cloud/index.md b/doc/ci/cloud_services/google_cloud/index.md index f4e4a2046ba..54265816868 100644 --- a/doc/ci/cloud_services/google_cloud/index.md +++ b/doc/ci/cloud_services/google_cloud/index.md @@ -67,7 +67,7 @@ inside the Workload Identity Pool created in the previous step, using the follow | `google.subject` | `assertion.sub` | | `attribute.X` | `assertion.X` | - You can also [build complex attributes](https://cloud.google.com/iam/help/workload-identity/attribute-mapping) + You can also [build complex attributes](https://cloud.google.com/iam/docs/workload-identity-federation#mapping) using Common Expression Language (CEL). You must map every attribute that you want to use for permission granting. For example, if you want to map permissions in the next step based on the user's email address, you must map `attribute.user_email` to `assertion.user_email`. diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index df0c7b69d46..ea4ad25637b 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -14,6 +14,9 @@ test it, and publish it to a container registry. To run Docker commands in your CI/CD jobs, you must configure GitLab Runner to support `docker` commands. +If you want to build Docker images without enabling privileged mode on the runner, +you can use a [Docker alternative](#docker-alternatives). + ## Enable Docker commands in your CI/CD jobs To enable Docker commands for your CI/CD jobs, you can use: @@ -22,9 +25,6 @@ To enable Docker commands for your CI/CD jobs, you can use: - [Docker-in-Docker](#use-docker-in-docker) - [Docker socket binding](#use-docker-socket-binding) -If you don't want to execute a runner in privileged mode, -but want to use `docker build`, you can also use [`kaniko`](using_kaniko.md) or [`buildah`](https://github.com/containers/buildah). - If you are using shared runners on GitLab.com, [learn more about how these runners are configured](../runners/index.md). @@ -133,7 +133,7 @@ To use Docker-in-Docker with TLS enabled: - This command registers a new runner to use the `docker:20.10.16` image. To start the build and service containers, it uses the `privileged` mode. - If you want to use [Docker-in-Docker](https://www.docker.com/blog/docker-can-now-run-within-docker/), + If you want to use Docker-in-Docker, you must always use `privileged = true` in your Docker containers. - This command mounts `/certs/client` for the service and build container, which is needed for the Docker client to use the @@ -345,8 +345,8 @@ not without its own challenges: root file system, you can use the job's working directory as a mount point for child containers. For example, if you have files you want to share with a child container, you might create a subdirectory under `/builds/$CI_PROJECT_PATH` - and use it as your mount point. For a more detailed explanation, view [issue - #41227](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227). + and use it as your mount point. For a more detailed explanation, view + [issue #41227](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227). ```yaml variables: @@ -406,8 +406,8 @@ sudo gitlab-runner register -n \ ##### Enable registry mirror for `docker:dind` service When the Docker daemon starts inside of the service container, it uses -the default configuration. You may want to configure a [registry -mirror](https://docs.docker.com/registry/recipes/mirror/) for +the default configuration. You may want to configure a +[registry mirror](https://docs.docker.com/registry/recipes/mirror/) for performance improvements and to ensure you don't reach Docker Hub rate limits. ###### The service in the `.gitlab-ci.yml` file @@ -831,6 +831,44 @@ If you're running multiple runners, you have to modify all configuration files. Read more about the [runner configuration](https://docs.gitlab.com/runner/configuration/) and [using the OverlayFS storage driver](https://docs.docker.com/engine/userguide/storagedriver/overlayfs-driver/). +## Docker alternatives + +To build Docker images without enabling privileged mode on the runner, you can +use one of these alternatives: + +- [`kaniko`](using_kaniko.md). +- [`buildah`](https://github.com/containers/buildah). + +For example, with `buildah`: + +```yaml +# Some details from https://major.io/2019/05/24/build-containers-in-gitlab-ci-with-buildah/ + +build: + stage: build + image: quay.io/buildah/stable + variables: + # Use vfs with buildah. Docker offers overlayfs as a default, but buildah + # cannot stack overlayfs on top of another overlayfs filesystem. + STORAGE_DRIVER: vfs + # Write all image metadata in the docker format, not the standard OCI format. + # Newer versions of docker can handle the OCI format, but older versions, like + # the one shipped with Fedora 30, cannot handle the format. + BUILDAH_FORMAT: docker + # You may need this workaround for some errors: https://stackoverflow.com/a/70438141/1233435 + BUILDAH_ISOLATION: chroot + FQ_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/test" + before_script: + # Log in to the GitLab container registry + - export REGISTRY_AUTH_FILE=${HOME}/auth.json + - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + script: + - buildah images + - buildah build -t $FQ_IMAGE_NAME + - buildah images + - buildah push $FQ_IMAGE_NAME +``` + ## Use the GitLab Container Registry After you've built a Docker image, you can push it up to the built-in diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index c2b18dc6467..712fd7b45d6 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -64,7 +64,7 @@ build: entrypoint: [""] script: - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 -w 0)\"}}}" > /kaniko/.docker/config.json - >- /kaniko/executor --context "${CI_PROJECT_DIR}" @@ -119,6 +119,13 @@ build: - if: $CI_COMMIT_TAG ``` +## Build a multi-arch image + +You can build [multi-arch images](https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/) +inside a container by using [`manifest-tool`](https://github.com/estesp/manifest-tool). + +For a detailed guide on how to build a multi-arch image, read [Building a multi-arch container image in unprivileged containers](https://ingenuity.siemens.com/2022/07/building-a-multi-arch-container-image-in-unprivileged-containers/). + ## Using a registry with a custom certificate When trying to push to a Docker registry that uses a certificate that is signed diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 6ffa68e4873..7747f5e9b78 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -21,7 +21,6 @@ GitLab: If you have a deployment service like [Kubernetes](../../user/infrastructure/clusters/index.md) associated with your project, you can use it to assist with your deployments. -You can even access a [web terminal](#web-terminals-deprecated) for your environment from within GitLab. ## View environments and deployments @@ -374,6 +373,9 @@ To retry or rollback a deployment: ### Environment URL +> - [Fixed](https://gitlab.com/gitlab-org/gitlab/-/issues/337417) to persist arbitrary URLs in GitLab 15.2 [with a flag](../../administration/feature_flags.md) named `soft_validation_on_external_url`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/337417) in GitLab 15.3. [Feature flag `soft_validation_on_external_url`](https://gitlab.com/gitlab-org/gitlab/-/issues/367206) removed. + The [environment URL](../yaml/index.md#environmenturl) is displayed in a few places in GitLab: diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 35ee4f9dd33..17eccc38747 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -70,7 +70,7 @@ Alternatively, you can use the API to protect an environment: name: ${CI_JOB_NAME} ``` -1. Use the UI to [create a new group](../../user/group/index.md#create-a-group). +1. Use the UI to [create a new group](../../user/group/manage.md#create-a-group). For example, this group is called `protected-access-group` and has the group ID `9899826`. Note that the rest of the examples in these steps use this group. @@ -189,11 +189,14 @@ and are protected at the same time. ### Configure group-level memberships +> - Operators are required to have Owner+ role from the original Maintainer+ role and this role change is introduced from GitLab 15.3 [with a flag](https://gitlab.com/gitlab-org/gitlab/-/issues/369873) named `group_level_protected_environment_settings_permission`. Enabled by default. +> - Original behavior where Operators are required to have Maintainer+ role can be achieved by enabling [flag](https://gitlab.com/gitlab-org/gitlab/-/issues/369875) named `override_group_level_protected_environment_settings_permission`. Disabled by default. + To maximize the effectiveness of group-level protected environments, [group-level memberships](../../user/group/index.md) must be correctly configured: -- Operators should be given at least the Maintainer role +- Operators should be given at least the Owner role for the top-level group. They can maintain CI/CD configurations for the higher environments (such as production) in the group-level settings page, which includes group-level protected environments, @@ -203,7 +206,7 @@ configured: This ensures that only operators can configure the organization-wide deployment ruleset. - Developers should be given no more than the Developer role - for the top-level group, or explicitly given the Maintainer role for a child project + for the top-level group, or explicitly given the Owner role for a child project They do *not* have access to the CI/CD configurations in the top-level group, so operators can ensure that the critical configuration won't be accidentally changed by the developers. diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 90cbcb9e240..00025a66936 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -180,13 +180,13 @@ $ vault write auth/jwt/role/myproject-production - <<EOF EOF ``` -This example uses [bound_claims](https://www.vaultproject.io/api/auth/jwt#bound_claims) to specify that only a JWT with matching values for the specified claims is allowed to authenticate. +This example uses [bound_claims](https://www.vaultproject.io/api-docs/auth/jwt#bound_claims) to specify that only a JWT with matching values for the specified claims is allowed to authenticate. Combined with [protected branches](../../../user/project/protected_branches.md), you can restrict who is able to authenticate and read the secrets. -[`token_explicit_max_ttl`](https://www.vaultproject.io/api/auth/jwt#token_explicit_max_ttl) specifies that the token issued by Vault, upon successful authentication, has a hard lifetime limit of 60 seconds. +[`token_explicit_max_ttl`](https://www.vaultproject.io/api-docs/auth/jwt#token_explicit_max_ttl) specifies that the token issued by Vault, upon successful authentication, has a hard lifetime limit of 60 seconds. -[`user_claim`](https://www.vaultproject.io/api/auth/jwt#user_claim) specifies the name for the Identity alias created by Vault upon a successful login. +[`user_claim`](https://www.vaultproject.io/api-docs/auth/jwt#user_claim) specifies the name for the Identity alias created by Vault upon a successful login. [`bound_claims_type`](https://www.vaultproject.io/api-docs/auth/jwt#bound_claims_type) configures the interpretation of the `bound_claims` values. If set to `glob`, the values are interpreted as globs, with `*` matching any number of characters. @@ -212,7 +212,7 @@ Role example to support the templated policy above, mapping the claim field `pro } ``` -For the full list of options, see Vault's [Create Role documentation](https://www.vaultproject.io/api/auth/jwt#create-role). +For the full list of options, see Vault's [Create Role documentation](https://www.vaultproject.io/api-docs/auth/jwt#create-role). WARNING: Always restrict your roles to project or namespace by using one of the provided claims (for example, `project_id` or `namespace_id`). Otherwise any JWT generated by this instance may be allowed to authenticate using this role. @@ -225,14 +225,15 @@ $ vault write auth/jwt/config \ bound_issuer="gitlab.example.com" ``` -[bound_issuer](https://www.vaultproject.io/api/auth/jwt#inlinecode-bound_issuer) specifies that only a JWT with the issuer (that is, the `iss` claim) set to `gitlab.example.com` can use this method to authenticate, and that the JWKS endpoint (`https://gitlab.example.com/-/jwks`) should be used to validate the token. +[bound_issuer](https://www.vaultproject.io/api-docs/auth/jwt#bound_issuer) specifies that only a JWT with the issuer (that is, the `iss` claim) set to `gitlab.example.com` can use this method to authenticate, and that the JWKS endpoint (`https://gitlab.example.com/-/jwks`) should be used to validate the token. -For the full list of available configuration options, see Vault's [API documentation](https://www.vaultproject.io/api/auth/jwt#configure). +For the full list of available configuration options, see Vault's [API documentation](https://www.vaultproject.io/api-docs/auth/jwt#configure). The following job, when run for the default branch, is able to read secrets under `secret/myproject/staging/`, but not the secrets under `secret/myproject/production/`: ```yaml read_secrets: + image: vault:latest script: # Check job's ref name - echo $CI_COMMIT_REF_NAME @@ -241,7 +242,7 @@ read_secrets: # Vault's address can be provided here or as CI/CD variable - export VAULT_ADDR=http://vault.example.com:8200 # Authenticate and get token. Token expiry time and other properties can be configured - # when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1 + # when configuring JWT Auth - https://www.vaultproject.io/api-docs/auth/jwt#parameters-1 - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-staging jwt=$CI_JOB_JWT)" # Now use the VAULT_TOKEN to read the secret and store it in an environment variable - export PASSWORD="$(vault kv get -field=password secret/myproject/staging/db)" @@ -261,6 +262,7 @@ The following job is able to authenticate using the `myproject-production` role ```yaml read_secrets: + image: vault:latest script: # Check job's ref name - echo $CI_COMMIT_REF_NAME @@ -269,7 +271,7 @@ read_secrets: # Vault's address can be provided here or as CI/CD variable - export VAULT_ADDR=http://vault.example.com:8200 # Authenticate and get token. Token expiry time and other properties can be configured - # when configuring JWT Auth - https://www.vaultproject.io/api/auth/jwt#parameters-1 + # when configuring JWT Auth - https://www.vaultproject.io/api-docs/auth/jwt#parameters-1 - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)" # Now use the VAULT_TOKEN to read the secret and store it in environment variable - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)" @@ -288,9 +290,9 @@ and GitLab features. For example, restrict the token by: for specific groups using `group_claim`. - Hard coding values for Vault bound claims based on the `user_login` and `user_email` of specific users. -- Setting Vault time limits for TTL of the token as specified in [`token_explicit_max_ttl`](https://www.vaultproject.io/api/auth/jwt#token_explicit_max_ttl), +- Setting Vault time limits for TTL of the token as specified in [`token_explicit_max_ttl`](https://www.vaultproject.io/api-docs/auth/jwt#token_explicit_max_ttl), where the token expires after authentication. -- Scoping the JWT to [GitLab projected branches](../../../user/project/protected_branches.md) +- Scoping the JWT to [GitLab protected branches](../../../user/project/protected_branches.md) that are restricted to a subset of project users. -- Scoping the JWT to [GitLab projected tags](../../../user/project/protected_tags.md), +- Scoping the JWT to [GitLab protected tags](../../../user/project/protected_tags.md), that are restricted to a subset of project users. diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 5f22fb894e5..fbecb1cd964 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -84,3 +84,9 @@ hidden files. If there is no `.gitmodules` file, it's possible the submodule settings are in a [`git config`](https://www.atlassian.com/git/tutorials/setting-up-a-repository/git-config) file. + +### `fatal: run_command returned non-zero status` error + +This error can happen in a job when working with submodules and the `GIT_STRATEGY` is set to `fetch`. + +Setting the `GIT_STRATEGY` to `clone` should resolve the issue. diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md index f49fdd6c39f..e6a9f1fa646 100644 --- a/doc/ci/interactive_web_terminal/index.md +++ b/doc/ci/interactive_web_terminal/index.md @@ -18,8 +18,8 @@ taken to protect the users. NOTE: [Shared runners on GitLab.com](../runners/index.md) do not -provide an interactive web terminal. Follow [this -issue](https://gitlab.com/gitlab-org/gitlab/-/issues/24674) for progress on +provide an interactive web terminal. Follow +[this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/24674) for progress on adding support. For groups and projects hosted on GitLab.com, interactive web terminals are available when using your own group or project runner. @@ -27,8 +27,8 @@ terminals are available when using your own group or project runner. Two things need to be configured for the interactive web terminal to work: -- The runner needs to have [`[session_server]` configured - properly](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section) +- The runner needs to have + [`[session_server]` configured properly](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-session_server-section) - If you are using a reverse proxy with your GitLab instance, web terminals need to be [enabled](../../administration/integration/terminal.md#enabling-and-disabling-terminal-support) @@ -54,8 +54,8 @@ Not all executors are NOTE: The `docker` executor does not keep running after the build script is finished. At that point, the terminal automatically -disconnects and does not wait for the user to finish. Please follow [this -issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3605) for updates on +disconnects and does not wait for the user to finish. Please follow +[this issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3605) for updates on improving this behavior. Sometimes, when a job is running, things don't go as you would expect, and it @@ -63,8 +63,7 @@ would be helpful if one can have a shell to aid debugging. When a job is running, on the right panel you can see a button `debug` that opens the terminal for the current job. - + When clicked, a new tab opens to the terminal page where you can access the terminal and type commands like a normal shell. diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index 9567bc9cd65..93f22da648a 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -13,6 +13,7 @@ You can use a GitLab CI/CD job token to authenticate with specific API endpoints - Packages: - [Package Registry](../../user/packages/package_registry/index.md#use-gitlab-cicd-to-build-packages). + - [Packages API](../../api/packages.md) (project-level). - [Container Registry](../../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) (the `$CI_REGISTRY_PASSWORD` is `$CI_JOB_TOKEN`). - [Container Registry API](../../api/container_registry.md) diff --git a/doc/ci/jobs/job_control.md b/doc/ci/jobs/job_control.md index 24133fe9a9b..0d5357e63ad 100644 --- a/doc/ci/jobs/job_control.md +++ b/doc/ci/jobs/job_control.md @@ -304,7 +304,7 @@ to add jobs to a pipeline: ```yaml docker build: variables: - DOCKERFILES_DIR: 'path/to/files/' + DOCKERFILES_DIR: 'path/to/files' script: docker build -t my-image:$CI_COMMIT_REF_SLUG . rules: - changes: @@ -567,6 +567,9 @@ In blocking manual jobs: enabled can't be merged with a blocked pipeline. - The pipeline shows a status of **blocked**. +When using manual jobs in triggered pipelines with [`strategy: depend`](../yaml/index.md#triggerstrategy), +the type of manual job can affect the trigger job's status while the pipeline runs. + ### Run a manual job To run a manual job, you must have permission to merge to the assigned branch: @@ -1005,6 +1008,26 @@ Additionally, rules with `changes` always evaluate as true in [scheduled pipelin All files are considered to have changed when a scheduled pipeline runs, so jobs might always be added to scheduled pipelines that use `changes`. +### File paths in CI/CD variables + +Be careful when using file paths in CI/CD variables. A trailing slash can appear correct +in the variable definition, but can become invalid when expanded in `script:`, `changes:`, +or other keywords. For example: + +```yaml +docker_build: + variables: + DOCKERFILES_DIR: 'path/to/files/' # This variable should not have a trailing '/' character + script: echo "A docker job" + rules: + - changes: + - $DOCKERFILES_DIR/* +``` + +When the `DOCKERFILES_DIR` variable is expanded in the `changes:` section, the full +path becomes `path/to/files//*`. The double slashes might cause unexpected behavior +depending on the keyword used, shell and OS of the runner, and so on. + ### `You are not allowed to download code from this project.` error message You might see pipelines fail when a GitLab administrator runs a protected manual job diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index f3044a03e04..3904a527a8f 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -116,8 +116,7 @@ you could add [`--no-tags`](https://git-scm.com/docs/git-fetch#Documentation/git to the extra flags to make your fetches faster and more compact. Also in the case where you repository does _not_ contain a lot of -tags, `--no-tags` can [make a big difference in some -cases](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/746). +tags, `--no-tags` can [make a big difference in some cases](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/746). If your CI builds do not depend on Git tags it is worth trying. See the [`GIT_FETCH_EXTRA_FLAGS` documentation](../runners/configure_runners.md#git-fetch-extra-flags) diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index 3b890458e56..7255d9aec82 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -265,7 +265,7 @@ test_async: ## Contexts and variables -CircleCI provides [Contexts](https://circleci.com/docs/2.0/contexts/) to securely pass environment variables across project pipelines. In GitLab, a [Group](../../user/group/index.md) can be created to assemble related projects together. At the group level, [CI/CD variables](../variables/index.md#add-a-cicd-variable-to-a-group) can be stored outside the individual projects, and securely passed into pipelines across multiple projects. +CircleCI provides [Contexts](https://circleci.com/docs/contexts) to securely pass environment variables across project pipelines. In GitLab, a [Group](../../user/group/index.md) can be created to assemble related projects together. At the group level, [CI/CD variables](../variables/index.md#add-a-cicd-variable-to-a-group) can be stored outside the individual projects, and securely passed into pipelines across multiple projects. ## Orbs diff --git a/doc/ci/pipeline_editor/index.md b/doc/ci/pipeline_editor/index.md index 4fb2ec94d60..0fd8fac7741 100644 --- a/doc/ci/pipeline_editor/index.md +++ b/doc/ci/pipeline_editor/index.md @@ -40,6 +40,10 @@ is invalid, a tip is shown to help you fix the problem: ## Lint CI configuration +NOTE: +The **Lint** tab is replaced with the **Validate** tab in GitLab 15.3. The lint results are included +in a successful [pipeline simulation](#simulate-a-cicd-pipeline). + To test the validity of your GitLab CI/CD configuration before committing the changes, you can use the CI lint tool. To access it, go to **CI/CD > Editor** and select the **Lint** tab. @@ -51,6 +55,15 @@ reflected in the CI lint. It displays the same results as the existing [CI Lint  +## Simulate a CI/CD pipeline + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/337282) in GitLab 15.3. + +To look for pipeline syntax and logic issues, you can simulate the creation of a +GitLab CI/CD pipeline in the **Validate** tab. A pipeline simulation can help find +problems such as incorrect `rules` and `needs` job dependencies, and is similar to +simulations in the [CI Lint tool](../lint.md#simulate-a-pipeline). + ## View included CI/CD configuration > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7064) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `pipeline_editor_file_tree`. Disabled by default. diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index 1e862c87035..4b7d3845361 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -21,7 +21,7 @@ On GitLab.com: - CI/CD minutes quotas are enabled for both public and private projects, but public projects [consume CI/CD minutes at a slower rate](#cost-factor). -- The base monthly CI/CD minutes quota for a GitLab.com [namespace](../../user/group/index.md#namespaces) +- The base monthly CI/CD minutes quota for a GitLab.com [namespace](../../user/namespace/index.md) is determined by its [license tier](https://about.gitlab.com/pricing/). - You can [purchase additional CI/CD minutes](#purchase-additional-cicd-minutes) if you need more than the number of CI/CD minutes in your monthly quota. @@ -204,12 +204,36 @@ The cost factors for jobs running on shared runners on GitLab.com are: - `0.008` for public projects, and projects in the [GitLab for Open Source program](../../subscriptions/index.md#gitlab-for-open-source). For every 125 minutes of job execution time, you use 1 CI/CD minute. - `1` for internal and private projects. +- Calculated differently for [community contributions to GitLab projects](#cost-factor-for-community-contributions-to-gitlab-projects). The cost factors on self-managed instances are: - `0` for public projects, so they do not consume CI/CD minutes. - `1` for internal and private projects. +#### Cost factor for community contributions to GitLab projects + +Community contributors can use up to 300,000 minutes on shared runners when +contributing to open source projects maintained by GitLab. The 300,000 +minutes applies to all SaaS tiers, and the cost factor calculation is: + +- `Monthly minute quota / 300,000 job duration minutes = Cost factor` + +For example, with the 10,000 CI/CD minutes per month in the Premium tier: + +- 10,000 / 300,000 = 0.03333333333 cost factor. + +For this reduced cost factor: + +- The merge request source project must be a fork of a GitLab-maintained project, + such as [`gitlab-com/www-gitlab-com`](https://gitlab.com/gitlab-com/www-gitlab-com), + [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab), and so on. +- The merge request target project must be the fork's parent project. +- The pipeline must be a merge request, merged results, or merge train pipeline. + +GitLab administrators can add a namespace to the reduced cost factor +[with a flag](../../administration/feature_flags.md) named `ci_minimal_cost_factor_for_gitlab_namespaces`. + ### Additional costs on GitLab SaaS GitLab SaaS shared runners have different cost factors, depending on the runner type (Linux, Windows, macOS) and the virtual machine configuration. diff --git a/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png b/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png Binary files differnew file mode 100644 index 00000000000..48a0ca9d84f --- /dev/null +++ b/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png diff --git a/doc/ci/pipelines/merge_request_pipelines.md b/doc/ci/pipelines/merge_request_pipelines.md index 89839de718b..5ba489c9830 100644 --- a/doc/ci/pipelines/merge_request_pipelines.md +++ b/doc/ci/pipelines/merge_request_pipelines.md @@ -151,7 +151,10 @@ or [**Rebase** option](../../user/project/merge_requests/methods/index.md#rebasi Prerequisites: -- You must be a member of the parent project and have at least the [Developer role](../../user/permissions.md). +- The parent project's [CI/CD configuration file](../yaml/index.md) must be configured to + [run jobs in merge request pipelines](#prerequisites). +- You must be a member of the parent project with [permissions to run CI/CD pipelines](../../user/permissions.md#gitlab-cicd-permissions). + You might need additional permissions if the branch is protected. - The fork project must be [visible](../../user/public_access.md) to the user running the pipeline. Otherwise, the **Pipelines** tab does not display in the merge request. @@ -161,6 +164,10 @@ To use the UI to run a pipeline in the parent project for a merge request from a 1. In the merge request, go to the **Pipelines** tab. 1. Select **Run pipeline**. You must read and accept the warning, or the pipeline does not run. +You can disable this feature by using [the projects API](../../api/projects.md#edit-project) +to disable the `ci_allow_fork_pipelines_to_run_in_parent_project` setting. +The setting is `enabled` by default. + ## Available predefined variables When you use merge request pipelines, you can use: diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index ac644628f3a..2882cd378aa 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -59,9 +59,8 @@ changes that are included in the target branch, and the `C` changes that are fro the merge request already in the train. <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -Watch this video for a demonstration on [how parallel execution -of merge trains can prevent commits from breaking the default -branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). +Watch this video for a demonstration on +[how parallel execution of merge trains can prevent commits from breaking the default branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). ## Prerequisites diff --git a/doc/ci/pipelines/multi_project_pipelines.md b/doc/ci/pipelines/multi_project_pipelines.md index dfbd2708d8d..a71af78f410 100644 --- a/doc/ci/pipelines/multi_project_pipelines.md +++ b/doc/ci/pipelines/multi_project_pipelines.md @@ -45,8 +45,8 @@ To create multi-project pipelines, you can: > [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) to GitLab Free in 12.8. -When you create a multi-project pipeline in your `.gitlab-ci.yml` file, -you create what is called a *trigger job*. For example: +When you use the [`trigger`](../yaml/index.md#trigger) keyword to create a multi-project +pipeline in your `.gitlab-ci.yml` file, you create what is called a *trigger job*. For example: ```yaml rspec: @@ -76,7 +76,7 @@ downstream project (`my/deployment`) too. If the downstream project is not found or the user does not have [permission](../../user/permissions.md) to create a pipeline there, the `staging` job is marked as _failed_. -#### Trigger job configuration keywords +#### Trigger job configuration limitations Trigger jobs can use only a limited set of the GitLab CI/CD [configuration keywords](../yaml/index.md). The keywords available for use in trigger jobs are: @@ -90,6 +90,8 @@ The keywords available for use in trigger jobs are: - [`extends`](../yaml/index.md#extends) - [`needs`](../yaml/index.md#needs), but not [`needs:project`](../yaml/index.md#needsproject) +Trigger jobs cannot use [job-level persisted variables](../variables/where_variables_can_be_used.md#persisted-variables). + #### Specify a downstream pipeline branch You can specify a branch name for the downstream pipeline to use. @@ -111,6 +113,8 @@ staging: Use: - The `project` keyword to specify the full path to a downstream project. + In [GitLab 15.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367660), variable expansion is + supported. - The `branch` keyword to specify the name of a branch in the project specified by `project`. In [GitLab 12.4 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/10126), variable expansion is supported. @@ -180,9 +184,12 @@ downstream-job: trigger: my/project ``` -In this scenario, the `UPSTREAM_BRANCH` variable with a value related to the -upstream pipeline is passed to the `downstream-job` job. It is available -in the context of all downstream builds. +In this scenario, the `UPSTREAM_BRANCH` variable with the value of the upstream pipeline's +`$CI_COMMIT_REF_NAME` is passed to `downstream-job`. It is available in the +context of all downstream builds. + +You cannot use this method to forward [job-level persisted variables](../variables/where_variables_can_be_used.md#persisted-variables) +to a downstream pipeline, as they are not available in trigger jobs. Upstream pipelines take precedence over downstream ones. If there are two variables with the same name defined in both upstream and downstream projects, @@ -228,6 +235,96 @@ In the upstream pipeline: artifacts: true ``` +#### Pass artifacts to a downstream pipeline + +You can pass artifacts to a downstream pipeline by using [`needs:project`](../yaml/index.md#needsproject). + +1. In a job in the upstream pipeline, save the artifacts using the [`artifacts`](../yaml/index.md#artifacts) keyword. +1. Trigger the downstream pipeline with a trigger job: + + ```yaml + build_artifacts: + stage: build + script: + - echo "This is a test artifact!" >> artifact.txt + artifacts: + paths: + - artifact.txt + + deploy: + stage: deploy + trigger: my/downstream_project + ``` + +1. In a job in the downstream pipeline, fetch the artifacts from the upstream pipeline + by using `needs:project`. Set `job` to the job in the upstream pipeline to fetch artifacts from, + `ref` to the branch, and `artifacts: true`. + + ```yaml + test: + stage: test + script: + - cat artifact.txt + needs: + - project: my/upstream_project + job: build_artifacts + ref: main + artifacts: true + ``` + +#### Pass artifacts to a downstream pipeline from a Merge Request pipeline + +When you use `needs:project` to [pass artifacts to a downstream pipeline](#pass-artifacts-to-a-downstream-pipeline), +the `ref` value is usually a branch name, like `main` or `development`. + +For merge request pipelines, the `ref` value is in the form of `refs/merge-requests/<id>/head`, +where `id` is the merge request ID. You can retrieve this ref with the [`CI_MERGE_REQUEST_REF_PATH`](../variables/predefined_variables.md#predefined-variables-for-merge-request-pipelines) +CI/CD variable. Do not use a branch name as the `ref` with merge request pipelines, +because the downstream pipeline attempts to fetch artifacts from the latest branch pipeline. + +To fetch the artifacts from the upstream `merge request` pipeline instead of the `branch` pipeline, +pass this variable to the downstream pipeline using variable inheritance: + +1. In a job in the upstream pipeline, save the artifacts using the [`artifacts`](../yaml/index.md#artifacts) keyword. +1. In the job that triggers the downstream pipeline, pass the `$CI_MERGE_REQUEST_REF_PATH` variable by using + [variable inheritance](#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword): + + ```yaml + build_artifacts: + stage: build + script: + - echo "This is a test artifact!" >> artifact.txt + artifacts: + paths: + - artifact.txt + + upstream_job: + variables: + UPSTREAM_REF: $CI_MERGE_REQUEST_REF_PATH + trigger: + project: my/downstream_project + branch: my-branch + ``` + +1. In a job in the downstream pipeline, fetch the artifacts from the upstream pipeline + by using `needs:project`. Set the `ref` to the `UPSTREAM_REF` variable, and `job` + to the job in the upstream pipeline to fetch artifacts from: + + ```yaml + test: + stage: test + script: + - cat artifact.txt + needs: + - project: my/upstream_project + job: build_artifacts + ref: UPSTREAM_REF + artifacts: true + ``` + +This method works for fetching artifacts from a regular merge request parent pipeline, +but fetching artifacts from [merge results](merged_results_pipelines.md) pipelines is not supported. + #### Use `rules` or `only`/`except` with multi-project pipelines You can use CI/CD variables or the [`rules`](../yaml/index.md#rulesif) keyword to @@ -244,8 +341,8 @@ If you use [`only/except`](../yaml/index.md#only--except) to control job behavio > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11238) in GitLab Premium 12.3. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/199224) to GitLab Free in 12.8. -You can mirror the pipeline status from the triggered pipeline to the source -trigger job by using `strategy: depend`. For example: +You can mirror the pipeline status from the triggered pipeline to the source trigger job +by using [`strategy: depend`](../yaml/index.md#triggerstrategy). For example: ```yaml trigger_job: @@ -303,11 +400,16 @@ downstream projects. On self-managed instances, an administrator can change this ## Multi-project pipeline visualization **(PREMIUM)** -When you configure GitLab CI/CD for your project, you can visualize the stages of your -[jobs](index.md#configure-a-pipeline) on a [pipeline graph](index.md#visualize-pipelines). +When your pipeline triggers a downstream pipeline, the downstream pipeline displays +to the right of the [pipeline graph](index.md#visualize-pipelines).  +In [pipeline mini graphs](index.md#pipeline-mini-graphs), the downstream pipeline +displays to the right of the mini graph. + + + ## Retry or cancel multi-project pipelines If you have permission to trigger pipelines in the downstream project, you can diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 43f20bfa9ea..34eae9828dd 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -250,8 +250,8 @@ using the [`coverage`](../yaml/index.md#coverage) keyword. ### Add test coverage results using project settings (removed) -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.8. Replaced by [`coverage` keyword](../yaml/index.md#coverage). -> [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.8. Replaced by [`coverage` keyword](../yaml/index.md#coverage). +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. This feature is in its end-of-life process. It was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 14.8. The feature is [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/17633) in GitLab 15.0. @@ -299,22 +299,22 @@ Use this regex for commonly used test tools. <!-- vale gitlab.Spelling = NO --> -- Simplecov (Ruby). Example: `\(\d+.\d+\%\) covered`. -- pytest-cov (Python). Example: `(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$`. -- Scoverage (Scala). Example: `Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)`. -- `phpunit --coverage-text --colors=never` (PHP). Example: `^\s*Lines:\s*\d+.\d+\%`. -- gcovr (C/C++). Example: `^TOTAL.*\s+(\d+\%)$`. -- `tap --coverage-report=text-summary` (NodeJS). Example: `^Statements\s*:\s*([^%]+)`. -- `nyc npm test` (NodeJS). Example: `All files[^|]*\|[^|]*\s+([\d\.]+)`. -- `jest --ci --coverage` (NodeJS). Example: `All files[^|]*\|[^|]*\s+([\d\.]+)`. -- excoveralls (Elixir). Example: `\[TOTAL\]\s+(\d+\.\d+)%`. -- `mix test --cover` (Elixir). Example: `\d+.\d+\%\s+\|\s+Total`. -- JaCoCo (Java/Kotlin). Example: `Total.*?([0-9]{1,3})%`. -- `go test -cover` (Go). Example: `coverage: \d+.\d+% of statements`. -- .NET (OpenCover). Example: `(Visited Points).*\((.*)\)`. -- .NET (`dotnet test` line coverage). Example: `Total\s*\|\s*(\d+(?:\.\d+)?)`. -- tarpaulin (Rust). Example: `^\d+.\d+% coverage`. -- Pester (PowerShell). Example: `Covered (\d+\.\d+%)`. +- Simplecov (Ruby). Example: `/\(\d+.\d+\%\) covered/`. +- pytest-cov (Python). Example: `/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/`. +- Scoverage (Scala). Example: `/Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)/`. +- `phpunit --coverage-text --colors=never` (PHP). Example: `/^\s*Lines:\s*\d+.\d+\%/`. +- gcovr (C/C++). Example: `/^TOTAL.*\s+(\d+\%)$/`. +- `tap --coverage-report=text-summary` (NodeJS). Example: `/^Statements\s*:\s*([^%]+)/`. +- `nyc npm test` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. +- `jest --ci --coverage` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. +- excoveralls (Elixir). Example: `/\[TOTAL\]\s+(\d+\.\d+)%/`. +- `mix test --cover` (Elixir). Example: `/\d+.\d+\%\s+\|\s+Total/`. +- JaCoCo (Java/Kotlin). Example: `/Total.*?([0-9]{1,3})%/`. +- `go test -cover` (Go). Example: `/coverage: \d+.\d+% of statements/`. +- .NET (OpenCover). Example: `/(Visited Points).*\((.*)\)/`. +- .NET (`dotnet test` line coverage). Example: `/Total\s*\|\s*(\d+(?:\.\d+)?)/`. +- tarpaulin (Rust). Example: `/^\d+.\d+% coverage/`. +- Pester (PowerShell). Example: `/Covered (\d+\.\d+%)/`. <!-- vale gitlab.Spelling = YES --> diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index efd78fac2c6..3efa697bf2f 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -722,11 +722,14 @@ variables: > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28940) in GitLab Runner 15.1. +NOTE: +Zip archives are the only supported artifact type. Follow [the issue for details](https://gitlab.com/gitlab-org/gitlab/-/issues/367203). + GitLab Runner can generate and produce attestation metadata for all build artifacts. To enable this feature, you must set the `RUNNER_GENERATE_ARTIFACTS_METADATA` environment variable to `true`. This variable can either be set globally or it can be set for individual jobs. The metadata is in rendered in a plain text `.json` file that's stored with the artifact. The file name is as follows: `{JOB_ID}-artifacts-metadata.json`. ### Attestation format -The attestation metadata is generated in the [in-toto attestation format](https://github.com/in-toto/attestation) for spec version [v0.1](https://in-toto.io/Statement/v0.1). The following fields are populated by default: +The attestation metadata is generated in the [in-toto attestation format](https://github.com/in-toto/attestation) for spec version [v0.1](https://github.com/in-toto/attestation/tree/v0.1.0/spec). The following fields are populated by default: | Field | Value | | ------ | ------ | @@ -863,7 +866,7 @@ Group runners are those that were created at the group level. ### View stale runner cleanup logs -You can check the [Sidekiq logs](../../administration/logs.md#sidekiq-logs) to see the cleanup result. In Kibana you can use the following query: +You can check the [Sidekiq logs](../../administration/logs/index.md#sidekiq-logs) to see the cleanup result. In Kibana you can use the following query: ```json { @@ -889,3 +892,23 @@ Filter entries where stale runners were removed: } } ``` + +## Determine which runners need to be upgraded **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365078) in GitLab 15.3. + +The version of GitLab Runner used by your runners should be +[kept up-to-date](https://docs.gitlab.com/runner/index.html#gitlab-runner-versions). + +To determine which runners need to be upgraded: + +1. View the list of runners: + - For a group, on the top bar, select **Menu > Groups** and on the left sidebar, select **CI/CD > Runners**. + - For the instance, select **Menu > Admin** and on the left sidebar, select **Runners**. + +1. Above the list of runners, view the status: + - **Outdated - recommended**: The runner does not have the latest `PATCH` version, which may make it vulnerable + to security or high severity bugs. Or, the runner is one or more `MAJOR` versions behind your GitLab instance, so some features may not be available or work properly. + - **Outdated - available**: Newer versions are available but upgrading is not critical. + +1. Filter the list by status to view which individual runners need to be upgraded. diff --git a/doc/ci/runners/img/build_isolation.png b/doc/ci/runners/img/build_isolation.png Binary files differnew file mode 100644 index 00000000000..a363ef4709b --- /dev/null +++ b/doc/ci/runners/img/build_isolation.png diff --git a/doc/ci/runners/index.md b/doc/ci/runners/index.md index 038bda4ab09..f69d1f0f730 100644 --- a/doc/ci/runners/index.md +++ b/doc/ci/runners/index.md @@ -18,5 +18,25 @@ The number of minutes you can use on these runners depends on the [maximum number of CI/CD minutes](../pipelines/cicd_minutes.md) in your [subscription plan](https://about.gitlab.com/pricing/). -If you use self-managed GitLab or you use GitLab.com but want to use your own runners, you can -[install and configure your own runners](https://docs.gitlab.com/runner/install/). +## Security for GitLab SaaS runners + +GitLab SaaS runners on Linux and Windows run on Google Compute Platform. The [Google Infrastructure Security Design Overview whitepaper](https://cloud.google.com/docs/security/infrastructure/design/resources/google_infrastructure_whitepaper_fa.pdf) provides an overview of how Google designs security into its technical infrastructure. The GitLab [Trust Center](https://about.gitlab.com/security/) and [GitLab Security Compliance Controls](https://about.staging.gitlab.com/handbook/engineering/security/security-assurance/security-compliance/sec-controls.html) pages provide an overview of the Security and Compliance controls that govern the GitLab SaaS runners. + +The runner that serves as a Runner Manager automatically initiates the creation and deletion of the virtual machines (VMs) used for CI jobs. When the Runner Manager picks up a GitLab SaaS CI job, it automatically executes that job on a new VM. There is no human or manual intervention in this process. The following section provides an overview of the additional built-in layers that harden the security of the GitLab Runner SaaS CI build environment. + +### Security of CI job execution on GitLab Runner SaaS (Linux, Windows) + +A dedicated temporary runner VM hosts and runs each CI job. On GitLab SaaS, two CI jobs never run on the same VM. + + + +In this example, there are three jobs in the project's pipeline. Therefore, there are three temporary VMs used to run that pipeline, or one VM per job. + +GitLab sends the command to remove the temporary runner VM to the Google Compute API immediately after the CI job completes. The [Google Compute Engine hypervisor](https://cloud.google.com/blog/products/gcp/7-ways-we-harden-our-kvm-hypervisor-at-google-cloud-security-in-plaintext) takes over the task of securely deleting the virtual machine and associated data. + +### Network security of CI job virtual machines on GitLab Runner SaaS (Linux, Windows) + +- Firewall rules only allow outbound communication from the temporary VM to the public internet. +- Inbound communication from the public internet to the temporary VM is not allowed. +- Firewall rules do not permit communication between VMs. +- The only internal communication allowed to the temporary VMs is from the Runner Manager. diff --git a/doc/ci/runners/runners_scope.md b/doc/ci/runners/runners_scope.md index 8b2753d26f1..9bd0b52f423 100644 --- a/doc/ci/runners/runners_scope.md +++ b/doc/ci/runners/runners_scope.md @@ -25,11 +25,11 @@ multiple projects. If you are using a self-managed instance of GitLab: - Your administrator can install and register shared runners by - going to your project's **Settings > CI/CD**, expanding the **Runners** section, - and clicking **Show runner installation instructions**. + going to your project's **Settings > CI/CD**, expanding **Runners**, + and selecting **Show runner installation instructions**. These instructions are also available [in the documentation](https://docs.gitlab.com/runner/install/index.html). -- The administrator can also configure a maximum number of shared runner [CI/CD minutes for - each group](../pipelines/cicd_minutes.md#set-the-quota-of-cicd-minutes-for-a-specific-namespace). +- The administrator can also configure a maximum number of shared runner + [CI/CD minutes for each group](../pipelines/cicd_minutes.md#set-the-quota-of-cicd-minutes-for-a-specific-namespace). If you are using GitLab.com: @@ -51,15 +51,19 @@ For existing projects, an administrator must To enable shared runners for a project: -1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. -1. Select **Enable shared runners for this project**. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. Turn on the **Enable shared runners for this project** toggle. ### Enable shared runners for a group To enable shared runners for a group: -1. Go to the group's **Settings > CI/CD** and expand the **Runners** section. -1. Select **Enable shared runners for this group**. +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. Turn on the **Enable shared runners for this group** toggle. ### Disable shared runners for a project @@ -69,8 +73,10 @@ or group. To disable shared runners for a project: -1. Go to the project's **Settings > CI/CD** and expand the **Runners** section. -1. In the **Shared runners** area, select **Enable shared runners for this project** so the toggle is grayed-out. +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. In the **Shared runners** area, turn off the **Enable shared runners for this project** toggle. Shared runners are automatically disabled for a project: @@ -81,9 +87,11 @@ Shared runners are automatically disabled for a project: To disable shared runners for a group: -1. Go to the group's **Settings > CI/CD** and expand the **Runners** section. -1. In the **Shared runners** area, turn off the **Enable shared runners for this group** toggle. -1. Optionally, to allow shared runners to be enabled for individual projects or subgroups, +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. Turn off the **Enable shared runners for this group** toggle. +1. Optional. To allow shared runners to be enabled for individual projects or subgroups, select **Allow projects and subgroups to override the group setting**. NOTE: @@ -147,7 +155,7 @@ The fair usage algorithm assigns jobs in this order: ## Group runners -Use *Group runners* when you want all projects in a group +Use _group runners_ when you want all projects in a group to have access to a set of runners. Group runners process jobs by using a first in, first out ([FIFO](https://en.wikipedia.org/wiki/FIFO_(computing_and_electronics))) queue. @@ -162,7 +170,7 @@ You must have the Owner role for the group. To create a group runner: 1. [Install GitLab Runner](https://docs.gitlab.com/runner/install/). -1. Go to the group you want to make the runner work for. +1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **CI/CD > Runners**. 1. Note the URL and token. 1. [Register the runner](https://docs.gitlab.com/runner/register/). @@ -175,21 +183,8 @@ You can view and manage all runners for a group, its subgroups, and projects. You can do this for your self-managed GitLab instance or for GitLab.com. You must have the Owner role for the group. -1. Go to the group where you want to view the runners. +1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **CI/CD > Runners**. -1. The following fields are displayed. - - | Attribute | Description | - | ------------ | ----------- | - | Type | Displays the runner type: `group` or `specific`, and the optional state `paused` | - | Runner token | Token used to identify the runner, and that the runner uses to communicate with the GitLab instance | - | Description | Description given to the runner when it was created | - | Version | GitLab Runner version | - | IP address | IP address of the host on which the runner is registered | - | Projects | The count of projects to which the runner is assigned | - | Jobs | Total of jobs run by the runner | - | Tags | Tags associated with the runner | - | Last contact | Timestamp indicating when the GitLab instance last contacted the runner | From this page, you can edit, pause, and remove runners from the group, its subgroups, and projects. @@ -198,7 +193,7 @@ From this page, you can edit, pause, and remove runners from the group, its subg You can pause or remove a group runner for your self-managed GitLab instance or for GitLab.com. You must have the Owner role for the group. -1. Go to the group you want to remove or pause the runner for. +1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **CI/CD > Runners**. 1. Select **Pause** or **Remove runner**. - If you pause a group runner that is used by multiple projects, the runner pauses for all projects. @@ -208,7 +203,7 @@ You must have the Owner role for the group. ## Specific runners -Use *Specific runners* when you want to use runners for specific projects. For example, +Use _specific runners_ when you want to use runners for specific projects. For example, when you have: - Jobs with specific requirements, like a deploy job that requires credentials. @@ -257,9 +252,8 @@ To enable a specific runner for a project: 1. On the top bar, select **Menu > Projects** and find the project where you want to enable the runner. 1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **General pipelines**. 1. Expand **Runners**. -1. By the runner you want, select **Enable for this project**. +1. In the **Specific runners** area, by the runner you want, select **Enable for this project**. You can edit a specific runner from any of the projects it's enabled for. The modifications, which include unlocking and editing tags and the description, @@ -275,9 +269,10 @@ but can also be changed later. To lock or unlock a specific runner: -1. Go to the project's **Settings > CI/CD**. -1. Expand the **Runners** section. +1. On the top bar, select **Menu > Projects** and find the project where you want to enable the runner. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. 1. Find the specific runner you want to lock or unlock. Make sure it's enabled. You cannot lock shared or group runners. 1. Select **Edit** (**{pencil}**). -1. Check the **Lock to current projects** option. +1. Select the **Lock to current projects** checkbox. 1. Select **Save changes**. diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index f1e28f7e74d..e96e89b47e5 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -24,7 +24,7 @@ The `gitlab-shared-runners-manager-X.gitlab.com` fleet of runners are dedicated Jobs handled by shared runners on GitLab.com (`shared-runners-manager-X.gitlab.com`) **time out after 3 hours**, regardless of the timeout configured in a -project. Check issue [#4010](https://gitlab.com/gitlab-com/infrastructure/-/issues/4010) and [#4070](https://gitlab.com/gitlab-com/infrastructure/-/issues/4070) for the reference. +project. Check issue [#4010](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4010) and [#4070](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4070) for the reference. Jobs handled by shared runners on Windows and macOS on GitLab.com **time out after 1 hour** while this service is in the [Beta](../../../policy/alpha-beta-support.md#beta-features) stage. diff --git a/doc/ci/runners/saas/windows_saas_runner.md b/doc/ci/runners/saas/windows_saas_runner.md index dddb3afee7c..f9fe6290220 100644 --- a/doc/ci/runners/saas/windows_saas_runner.md +++ b/doc/ci/runners/saas/windows_saas_runner.md @@ -126,8 +126,7 @@ test: ## Limitations and known issues -- All the limitations mentioned in our [beta - definition](../../../policy/alpha-beta-support.md#beta-features). +- All the limitations mentioned in our [beta definition](../../../policy/alpha-beta-support.md#beta-features). - The average provisioning time for a new Windows VM is 5 minutes. This means that you may notice slower build start times on the Windows runner fleet during the beta. In a future diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index ba395108966..fb91aeb6240 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -88,7 +88,7 @@ To configure your Vault server: - `VAULT_SERVER_URL` - The URL of your Vault server, such as `https://vault.example.com:8200`. Required. - `VAULT_AUTH_ROLE` - Optional. The role to use when attempting to authenticate. - If no role is specified, Vault uses the [default role](https://www.vaultproject.io/api/auth/jwt#default_role) + If no role is specified, Vault uses the [default role](https://www.vaultproject.io/api-docs/auth/jwt#default_role) specified when the authentication method was configured. - `VAULT_AUTH_PATH` - Optional. The path where the authentication method is mounted, default is `jwt`. - `VAULT_NAMESPACE` - Optional. The [Vault Enterprise namespace](https://www.vaultproject.io/docs/enterprise/namespaces) to use for reading secrets and authentication. @@ -183,7 +183,7 @@ For a full list of `CI_JOB_JWT` claims, read the You can also specify some attributes for the resulting Vault tokens, such as time-to-live, IP address range, and number of uses. The full list of options is available in -[Vault's documentation on creating roles](https://www.vaultproject.io/api/auth/jwt#create-role) +[Vault's documentation on creating roles](https://www.vaultproject.io/api-docs/auth/jwt#create-role) for the JSON web token method. ## Using a self-signed Vault server diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index fb421ab8944..8e141c62ef6 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -36,7 +36,7 @@ To add a secure file to a project: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > CI/CD**. -1. In the **Secure Files** section, select **Manage**. +1. In the **Secure Files** section, select **Expand**. 1. Select **Upload File**. 1. Find the file to upload, select **Open**, and the file upload begins immediately. The file shows up in the list when the upload is complete. diff --git a/doc/ci/testing/load_performance_testing.md b/doc/ci/testing/load_performance_testing.md index ae177958beb..e15b3944c2b 100644 --- a/doc/ci/testing/load_performance_testing.md +++ b/doc/ci/testing/load_performance_testing.md @@ -121,7 +121,7 @@ the k6 test. NOTE: For Kubernetes setups a different template should be used: [`Jobs/Load-Performance-Testing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml). -k6 has [various options](https://k6.io/docs/using-k6/options) to configure how it will run tests, such as what throughput (RPS) to run with, +k6 has [various options](https://k6.io/docs/using-k6/k6-options/reference/) to configure how it will run tests, such as what throughput (RPS) to run with, how long the test should run, and so on. Almost all options can be configured in the test itself, but as you can also pass command line options via the `K6_OPTIONS` variable. diff --git a/doc/ci/testing/unit_test_report_examples.md b/doc/ci/testing/unit_test_report_examples.md index a54deb254b7..b49ac29be65 100644 --- a/doc/ci/testing/unit_test_report_examples.md +++ b/doc/ci/testing/unit_test_report_examples.md @@ -250,7 +250,7 @@ test: This example uses [PHPUnit](https://phpunit.de/) with the `--log-junit` flag. You can also add this option using -[XML](https://phpunit.readthedocs.io/en/stable/configuration.html#the-junit-element) +[XML](https://phpunit.readthedocs.io/en/9.5/configuration.html#the-junit-element) in the `phpunit.xml` configuration file. ```yaml diff --git a/doc/ci/testing/unit_test_reports.md b/doc/ci/testing/unit_test_reports.md index c8e0d6135df..6294996c703 100644 --- a/doc/ci/testing/unit_test_reports.md +++ b/doc/ci/testing/unit_test_reports.md @@ -46,24 +46,44 @@ comparing the head and base branch's JUnit report format XML files, where: - The base branch is the target branch (usually the default branch). - The head branch is the source branch (the latest pipeline in each merge request). -The reports panel has a summary showing how many tests failed, how many had errors +The **Test summary** panel shows how many tests failed, how many had errors, and how many were fixed. If no comparison can be done because data for the base branch -is not available, the panel just shows the list of failed tests for head. +is not available, the panel shows only the list of failed tests for the source branch. -There are four types of results: +The types of results are: -1. **Newly failed tests:** Test cases which passed on base branch and failed on head branch -1. **Newly encountered errors:** Test cases which passed on base branch and failed due to a - test error on head branch -1. **Existing failures:** Test cases which failed on base branch and failed on head branch -1. **Resolved failures:** Test cases which failed on base branch and passed on head branch +- **Newly failed tests:** Test cases which passed on the base branch and failed on the head branch. +- **Newly encountered errors:** Test cases which passed on the base branch and failed due to a + test error on the head branch. +- **Existing failures:** Test cases which failed on the base branch and failed on the head branch. +- **Resolved failures:** Test cases which failed on the base branch and passed on the head branch. -Each entry in the panel shows the test name and its type from the list -above. Clicking on the test name opens a modal window with details of its -execution time and the error output. +### View failed tests + +Each entry in the **Test summary** panel shows the test name and result type. +Select the test name to open a modal window with details of its execution time and +the error output.  +#### Copy failed test names + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91552) in GitLab 15.2. + +You can copy the name and path of failed tests when there are failed tests listed +in the **Test summary** panel. Use name and path to find and rerun the +test locally for verification. + +To copy the name of all failed tests, at the top of the **Test summary** panel, +select **Copy failed tests**. The failed tests are listed as a string with the tests +separated by spaces. + +To copy the name of a single failed test: + +1. Expand the **Test summary** panel by selecting **Show test summary details** (**{chevron-lg-down}**). +1. Select the test you want to review. +1. Select **Copy test name to rerun locally** (**{copy-to-clipboard}**). + ### Number of recent failures > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241759) in merge requests in GitLab 13.7. diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 6df614c1cda..f8900501244 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -82,7 +82,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_PAGES_URL` | 11.8 | all | The URL for a GitLab Pages site. Always a subdomain of `CI_PAGES_DOMAIN`. | | `CI_PIPELINE_ID` | 8.10 | all | The instance-level ID of the current pipeline. This ID is unique across all projects on the GitLab instance. | | `CI_PIPELINE_IID` | 11.0 | all | The project-level IID (internal ID) of the current pipeline. This ID is unique only within the current project. | -| `CI_PIPELINE_SOURCE` | 10.0 | all | How the pipeline was triggered. Can be `push`, `web`, `schedule`, `api`, `external`, `chat`, `webide`, `merge_request_event`, `external_pull_request_event`, `parent_pipeline`, [`trigger`, or `pipeline`](../triggers/index.md#configure-cicd-jobs-to-run-in-triggered-pipelines). | +| `CI_PIPELINE_SOURCE` | 10.0 | all | How the pipeline was triggered. Can be `push`, `web`, `schedule`, `api`, `external`, `chat`, `webide`, `merge_request_event`, `external_pull_request_event`, `parent_pipeline`, [`trigger`, or `pipeline`](../triggers/index.md#configure-cicd-jobs-to-run-in-triggered-pipelines). For a description of each value, see [Common `if` clauses for `rules`](../jobs/job_control.md#common-if-clauses-for-rules), which uses this variable to control when jobs run. | | `CI_PIPELINE_TRIGGERED` | all | all | `true` if the job was [triggered](../triggers/index.md). | | `CI_PIPELINE_URL` | 11.1 | 0.5 | The URL for the pipeline details. | | `CI_PIPELINE_CREATED_AT` | 13.10 | all | The UTC datetime when the pipeline was created, in [ISO 8601](https://tools.ietf.org/html/rfc3339#appendix-A) format. | @@ -93,7 +93,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_PROJECT_NAMESPACE` | 8.10 | 0.5 | The project namespace (username or group name) of the job. | | `CI_PROJECT_PATH_SLUG` | 9.3 | all | `$CI_PROJECT_PATH` in lowercase with characters that are not `a-z` or `0-9` replaced with `-` and shortened to 63 bytes. Use in URLs and domain names. | | `CI_PROJECT_PATH` | 8.10 | 0.5 | The project namespace with the project name included. | -| `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | A comma-separated, lowercase list of the languages used in the repository. For example `ruby,javascript,html,css`. | +| `CI_PROJECT_REPOSITORY_LANGUAGES` | 12.3 | all | A comma-separated, lowercase list of the languages used in the repository. For example `ruby,javascript,html,css`. The maximum number of languages is limited to 5. An issue [proposes to increase the limit](https://gitlab.com/gitlab-org/gitlab/-/issues/368925). | | `CI_PROJECT_ROOT_NAMESPACE` | 13.2 | 0.5 | The root project namespace (username or group name) of the job. For example, if `CI_PROJECT_NAMESPACE` is `root-group/child-group/grandchild-group`, `CI_PROJECT_ROOT_NAMESPACE` is `root-group`. | | `CI_PROJECT_TITLE` | 12.4 | all | The human-readable project name as displayed in the GitLab web interface. | | `CI_PROJECT_DESCRIPTION` | 15.1 | all | The project description as displayed in the GitLab web interface. | @@ -124,6 +124,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_SERVER_VERSION` | all | all | The full version of the GitLab instance. | | `CI_SERVER` | all | all | Available for all jobs executed in CI/CD. `yes` when available. | | `CI_SHARED_ENVIRONMENT` | all | 10.1 | Only available if the job is executed in a shared environment (something that is persisted across CI/CD invocations, like the `shell` or `ssh` executor). `true` when available. | +| `CI_TEMPLATE_REGISTRY_HOST` | 15.3 | all | The host of the registry used by CI/CD templates. Defaults to `registry.gitlab.com`. | | `GITLAB_CI` | all | all | Available for all jobs executed in CI/CD. `true` when available. | | `GITLAB_FEATURES` | 10.6 | all | The comma-separated list of licensed features available for the GitLab instance and license. | | `GITLAB_USER_EMAIL` | 8.12 | all | The email of the user who started the job. | diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 7a1ef61f109..db83f2a14f3 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -40,6 +40,7 @@ There are two places defined variables can be used. On the: | [`services:name`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | | [`services`](../yaml/index.md#services) | yes | Runner | The variable expansion is made by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | | [`tags`](../yaml/index.md#tags) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35742) in GitLab 14.1. | +| [`trigger` and `trigger:project`](../yaml/index.md#trigger) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367660) in GitLab 15.3. | | [`variables`](../yaml/index.md#variables) | yes | GitLab/Runner | The variable expansion is first made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab, and then any unrecognized or unavailable variables are expanded by GitLab Runner's [internal variable expansion mechanism](#gitlab-runner-internal-variable-expansion-mechanism). | ### `config.toml` file @@ -131,10 +132,17 @@ These restrictions exist because `after_script` scripts are executed in a ## Persisted variables -The following variables are known as "persisted": +Some predefined variables are called "persisted". + +Pipeline-level persisted variables: - `CI_PIPELINE_ID` +- `CI_PIPELINE_URL` + +Job-level persisted variables: + - `CI_JOB_ID` +- `CI_JOB_URL` - `CI_JOB_TOKEN` - `CI_JOB_STARTED_AT` - `CI_REGISTRY_USER` @@ -143,7 +151,7 @@ The following variables are known as "persisted": - `CI_DEPLOY_USER` - `CI_DEPLOY_PASSWORD` -They are: +Persisted variables are: - Supported for definitions where the ["Expansion place"](#gitlab-ciyml-file) is: - Runner. @@ -152,6 +160,9 @@ They are: - For definitions where the ["Expansion place"](#gitlab-ciyml-file) is GitLab. - In the `only`, `except`, and `rules` [variables expressions](../jobs/job_control.md#cicd-variable-expressions). +[Pipeline trigger jobs](../yaml/index.md#trigger) cannot use job-level persisted variables, +but can use pipeline-level persisted variables. + Some of the persisted variables contain tokens and cannot be used by some definitions due to security reasons. diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index 379a4b3224a..61ef8bbfab7 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -329,3 +329,27 @@ GitLab can display the results of one or more reports in the merge request [terraform widget](../../user/infrastructure/iac/mr_integration.md#output-terraform-plan-information-into-a-merge-request). For more information, see [Output `terraform plan` information into a merge request](../../user/infrastructure/iac/mr_integration.md). + +## `artifacts:reports:cyclonedx` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360766) in GitLab 15.3 + +This report is a Software Bill of Materials describing the components of a project +following the [cyclonedx](https://cyclonedx.org/docs/1.4) protocol format. + +You can specify multiple cyclonedx reports per job. These can be either supplied +as a list of filenames, a filename pattern, or both: + +- List of filenames: `cyclonedx: [gl-sbom-npm-npm.cdx.json, gl-sbom-bundler-gem.cdx.json]`. +- A filename pattern: `cyclonedx: gl-sbom-*.json`. +- Combination of both of the above: `cyclonedx: [gl-sbom-*.json, my-cyclonedx.json]`. + +Below is an example of a job exposing cyclonedx artifacts: + +```yaml +artifacts: + reports: + cyclonedx: + - gl-sbom-npm-npm.cdx.json + - gl-sbom-bundler-gem.cdx.json +``` diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index 838710834ba..5158f80ea64 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -289,10 +289,10 @@ smoke-test-job: In `include` sections in your `.gitlab-ci.yml` file, you can use: -- [Project variables](../variables/index.md#add-a-cicd-variable-to-a-project) -- [Group variables](../variables/index.md#add-a-cicd-variable-to-a-group) -- [Instance variables](../variables/index.md#add-a-cicd-variable-to-an-instance) -- Project [predefined variables](../variables/predefined_variables.md) +- [Project variables](../variables/index.md#add-a-cicd-variable-to-a-project). +- [Group variables](../variables/index.md#add-a-cicd-variable-to-a-group). +- [Instance variables](../variables/index.md#add-a-cicd-variable-to-an-instance). +- Project [predefined variables](../variables/predefined_variables.md). - In GitLab 14.2 and later, the `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md). When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full @@ -322,7 +322,11 @@ include: file: '.compliance-gitlab-ci.yml' ``` -For an example of how you can include these predefined variables, and the variables' impact on CI/CD jobs, +You cannot use variables defined in jobs, or in a global [`variables`](../yaml/index.md#variables) +section which defines the default variables for all jobs. Includes are evaluated before jobs, +so these variables cannot be used with `include`. + +For an example of how you can include predefined variables, and the variables' impact on CI/CD jobs, see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos). ## Use `rules` with `include` diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 3bb2007d6e3..0b434e7013c 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -171,11 +171,11 @@ Use `include:local` instead of symbolic links. **Possible inputs**: -- A full path relative to the root directory (`/`). +A full path relative to the root directory (`/`): + - The YAML file must have the extension `.yml` or `.yaml`. - You can [use `*` and `**` wildcards in the file path](includes.md#use-includelocal-with-wildcard-file-paths). - -CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). +- You can use [certain CI/CD variables](includes.md#use-variables-with-include). **Example of `include:local`**: @@ -208,10 +208,10 @@ use `include:file`. You can use `include:file` in combination with `include:proj **Possible inputs**: -- A full path, relative to the root directory (`/`). The YAML file must have the - extension `.yml` or `.yaml`. +A full path, relative to the root directory (`/`): -CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). +- The YAML file must have the extension `.yml` or `.yaml`. +- You can use [certain CI/CD variables](includes.md#use-variables-with-include). **Example of `include:file`**: @@ -268,10 +268,11 @@ Use `include:remote` with a full URL to include a file from a different location **Possible inputs**: -- A public URL accessible by an HTTP/HTTPS `GET` request. Authentication with the - remote URL is not supported. The YAML file must have the extension `.yml` or `.yaml`. +A public URL accessible by an HTTP/HTTPS `GET` request: -CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). +- Authentication with the remote URL is not supported. +- The YAML file must have the extension `.yml` or `.yaml`. +- You can use [certain CI/CD variables](includes.md#use-variables-with-include). **Example of `include:remote`**: @@ -296,9 +297,12 @@ Use `include:template` to include [`.gitlab-ci.yml` templates](https://gitlab.co **Possible inputs**: -- [`.gitlab-ci.yml` templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). +A [CI/CD template](../examples/index.md#cicd-templates): -CI/CD variables [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). +- Templates are stored in [`lib/gitlab/ci/templates`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). + Not all templates are designed to be used with `include:template`, so check template + comments before using one. +- You can use [certain CI/CD variables](includes.md#use-variables-with-include). **Example of `include:template`**: @@ -2136,7 +2140,7 @@ This example creates four paths of execution: - The maximum number of jobs that a single job can have in the `needs` array is limited: - For GitLab.com, the limit is 50. For more information, see our - [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7541). + [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/7541). - For self-managed instances, the default limit is 50. This limit [can be changed](../../administration/cicd.md#set-the-needs-job-limit). - If `needs` refers to a job that uses the [`parallel`](#parallel) keyword, it depends on all jobs created in parallel, not just one job. It also downloads @@ -2784,11 +2788,15 @@ which must be in the `$PATH`. If you use the [Docker executor](https://docs.gitlab.com/runner/executors/docker.html), you can use this image from the GitLab Container Registry: `registry.gitlab.com/gitlab-org/release-cli:latest` +If you use the [Shell executor](https://docs.gitlab.com/runner/executors/shell.html) or similar, +[install `release-cli`](../../user/project/releases/release_cli.md) on the server where the runner is registered. + **Keyword type**: Job keyword. You can use it only as part of a job. **Possible inputs**: The `release` subkeys: - [`tag_name`](#releasetag_name) +- [`tag_message`](#releasetag_message) (optional) - [`name`](#releasename) (optional) - [`description`](#releasedescription) - [`ref`](#releaseref) (optional) @@ -2832,8 +2840,6 @@ This example creates a release: - The `release` section executes after the `script` keyword and before the `after_script`. - A release is created only if the job's main script succeeds. - If the release already exists, it is not updated and the job with the `release` keyword fails. -- If you use the [Shell executor](https://docs.gitlab.com/runner/executors/shell.html) or similar, - [install `release-cli`](../../user/project/releases/release_cli.md) on the server where the runner is registered. **Related topics**: @@ -2887,6 +2893,30 @@ job: - if: $CI_PIPELINE_SOURCE == "schedule" ``` +#### `release:tag_message` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363024) in GitLab 15.3. Supported by `release-cli` v0.12.0 or later. + +If the tag does not exist, the newly created tag is annotated with the message specifed by `tag_message`. +If omitted, a lightweight tag is created. + +**Keyword type**: Job keyword. You can use it only as part of a job. + +**Possible inputs**: + +- A text string. + +**Example of `release:tag_message`**: + +```yaml + release_job: + stage: release + release: + tag_name: $CI_COMMIT_TAG + description: 'Release description' + tag_message: 'Annotated tag message' +``` + #### `release:name` The release name. If omitted, it is populated with the value of `release: tag_name`. @@ -2965,7 +2995,7 @@ released_at: '2021-03-15T08:00:00Z' > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/271454) in GitLab 13.12. -Use `release:assets:links` to include [asset links](../../user/project/releases/index.md#release-assets) in the release. +Use `release:assets:links` to include [asset links](../../user/project/releases/release_fields.md#release-assets) in the release. Requires `release-cli` version v0.4.0 or later. @@ -3221,6 +3251,7 @@ branch or merge request pipelines. **Possible inputs**: - An array of file paths. In GitLab 13.6 and later, [file paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). +- Alternatively, the array of file paths can be in [`rules:changes:paths`](#ruleschangespaths). **Example of `rules:changes`**: @@ -3239,6 +3270,8 @@ docker build: - If `Dockerfile` has changed, add the job to the pipeline as a manual job, and the pipeline continues running even if the job is not triggered (`allow_failure: true`). - If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). +- [`rules:changes:paths`](#ruleschangespaths) is the same as `rules:changes` without + any subkeys. **Additional details**: @@ -3250,6 +3283,74 @@ docker build: - [Jobs or pipelines can run unexpectedly when using `rules: changes`](../jobs/job_control.md#jobs-or-pipelines-run-unexpectedly-when-using-changes). +##### `rules:changes:paths` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90171) in GitLab 15.2. + +Use `rules:changes` to specify that a job only be added to a pipeline when specific +files are changed, and use `rules:changes:paths` to specify the files. + +`rules:changes:paths` is the same as using [`rules:changes`](#ruleschanges) without +any subkeys. All additional details and related topics are the same. + +**Keyword type**: Job keyword. You can use it only as part of a job. + +**Possible inputs**: + +- An array of file paths. In GitLab 13.6 and later, [file paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). + +**Example of `rules:changes:paths`**: + +```yaml +docker-build-1: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + - Dockerfile + +docker-build-2: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + paths: + - Dockerfile +``` + +In this example, both jobs have the same behavior. + +##### `rules:changes:compare_to` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/293645) in GitLab 15.3 [with a flag](../../administration/feature_flags.md) named `ci_rules_changes_compare`. Enabled by default. + +Use `rules:changes:compare_to` to specify which ref to compare against for changes to the files +listed under [`rules:changes:paths`](#ruleschangespaths). + +**Keyword type**: Job keyword. You can use it only as part of a job, and it must be combined with `rules:changes:paths`. + +**Possible inputs**: + +- A branch name, like `main`, `branch1`, or `refs/heads/branch1`. +- A tag name, like `tag1` or `refs/tags/tag1`. +- A commit SHA, like `2fg31ga14b`. + +**Example of `rules:changes:compare_to`**: + +```yaml +docker build: + script: docker build -t my-image:$CI_COMMIT_REF_SLUG . + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + changes: + paths: + - Dockerfile + compare_to: 'refs/heads/branch1' +``` + +In this example, the `docker build` job is only included when the `Dockerfile` has changed +relative to `refs/heads/branch1` and the pipeline source is a merge request event. + #### `rules:exists` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/24021) in GitLab 12.4. @@ -3792,7 +3893,9 @@ Use `trigger` to start a downstream pipeline that is either: **Possible inputs**: -- For multi-project pipelines, path to the downstream project. +- For multi-project pipelines, path to the downstream project. CI/CD variables + [are supported](../variables/where_variables_can_be_used.md#gitlab-ciyml-file) + in GitLab 15.3 and later. - For child pipelines, path to the child pipeline CI/CD configuration file. **Example of `trigger` for multi-project pipeline**: @@ -3830,6 +3933,8 @@ trigger_job: and [scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule) are not passed to downstream pipelines by default. Use [trigger:forward](#triggerforward) to forward these variables to downstream pipelines. +- [Job-level persisted variables](../variables/where_variables_can_be_used.md#persisted-variables) + are not available in trigger jobs. **Related topics**: @@ -3861,6 +3966,17 @@ trigger_job: In this example, jobs from subsequent stages wait for the triggered pipeline to successfully complete before starting. +**Additional details**: + +- [Optional manual jobs](../jobs/job_control.md#types-of-manual-jobs) in the downstream pipeline + do not affect the status of the downstream pipeline or the upstream trigger job. + The downstream pipeline can complete successfully without running any optional manual jobs. +- [Blocking manual jobs](../jobs/job_control.md#types-of-manual-jobs) in the downstream pipeline + must run before the trigger job is marked as successful or failed. The trigger job + shows **pending** (**{status_pending}**) if the downstream pipeline status is + **waiting for manual action** (**{status_manual}**) due to manual jobs. By default, + jobs in later stages do not start until the trigger job completes. + #### `trigger:forward` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213729) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `ci_trigger_forward_variables`. Disabled by default. diff --git a/doc/ci/yaml/script.md b/doc/ci/yaml/script.md index 4bffcbca1cc..f1cdcf57e64 100644 --- a/doc/ci/yaml/script.md +++ b/doc/ci/yaml/script.md @@ -258,3 +258,27 @@ pages-job: script: - 'curl --header "PRIVATE-TOKEN: ${PRIVATE_TOKEN}" "https://gitlab.example.com/api/v4/projects"' ``` + +### Job does not fail when using `&&` in a script + +If you use `&&` to combine two commands together in a single script line, the job +might return as successful, even if one of the commands failed. For example: + +```yaml +job-does-not-fail: + script: + - invalid-command xyz && invalid-command abc + - echo $? + - echo "The job should have failed already, but this is executed unexpectedly." +``` + +The `&&` operator returns an exit code of `0` even though the two commands failed, +and the job continues to run. To force the script to exit when either command fails, +enclose the entire line in parentheses: + +```yaml +job-fails: + script: + - (invalid-command xyz && invalid-command abc) + - echo "The job failed already, and this is not executed." +``` diff --git a/doc/development/adding_database_indexes.md b/doc/development/adding_database_indexes.md index e80bffe7c18..7ab846cce3e 100644 --- a/doc/development/adding_database_indexes.md +++ b/doc/development/adding_database_indexes.md @@ -1,314 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/adding_database_indexes.md' +remove_date: '2022-11-05' --- -# Adding Database Indexes +This document was moved to [another location](database/adding_database_indexes.md). -Indexes can be used to speed up database queries, but when should you add a new -index? Traditionally the answer to this question has been to add an index for -every column used for filtering or joining data. For example, consider the -following query: - -```sql -SELECT * -FROM projects -WHERE user_id = 2; -``` - -Here we are filtering by the `user_id` column and as such a developer may decide -to index this column. - -While in certain cases indexing columns using the above approach may make sense, -it can actually have a negative impact. Whenever you write data to a table, any -existing indexes must also be updated. The more indexes there are, the slower this -can potentially become. Indexes can also take up significant disk space, depending -on the amount of data indexed and the index type. For example, PostgreSQL offers -`GIN` indexes which can be used to index certain data types that cannot be -indexed by regular B-tree indexes. These indexes, however, generally take up more -data and are slower to update compared to B-tree indexes. - -Because of all this, it's important make the following considerations -when adding a new index: - -1. Do the new queries re-use as many existing indexes as possible? -1. Is there enough data that using an index is faster than iterating over - rows in the table? -1. Is the overhead of maintaining the index worth the reduction in query - timings? - -## Re-using Queries - -The first step is to make sure your query re-uses as many existing indexes as -possible. For example, consider the following query: - -```sql -SELECT * -FROM todos -WHERE user_id = 123 -AND state = 'open'; -``` - -Now imagine we already have an index on the `user_id` column but not on the -`state` column. One may think this query performs badly due to `state` being -unindexed. In reality the query may perform just fine given the index on -`user_id` can filter out enough rows. - -The best way to determine if indexes are re-used is to run your query using -`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering, -you may find an extra index doesn't make much, if any, difference. - -In short: - -1. Try to write your query in such a way that it re-uses as many existing - indexes as possible. -1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most - ideal query. - -## Data Size - -A database may not use an index even when a regular sequence scan -(iterating over all rows) is faster, especially for small tables. - -Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows. -You may _not_ want to add an index if the table size is small (<`1,000` records), -or if existing indexes already filter out enough rows. - -## Maintenance Overhead - -Indexes have to be updated on every table write. In the case of PostgreSQL, _all_ -existing indexes are updated whenever data is written to a table. As a -result, having many indexes on the same table slows down writes. It's therefore important -to balance query performance with the overhead of maintaining an extra index. - -Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases -INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth -it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE -timings are unaffected. - -## Finding Unused Indexes - -To see which indexes are unused you can run the following query: - -```sql -SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass)) -FROM pg_stat_all_indexes -WHERE schemaname = 'public' -AND idx_scan = 0 -AND idx_tup_read = 0 -AND idx_tup_fetch = 0 -ORDER BY pg_relation_size(indexrelname::regclass) desc; -``` - -This query outputs a list containing all indexes that are never used and sorts -them by indexes sizes in descending order. This query helps in -determining whether existing indexes are still required. More information on -the meaning of the various columns can be found at -<https://www.postgresql.org/docs/current/monitoring-stats.html>. - -To determine if an index is still being used on production, use the following -Thanos query with your index name: - -```sql -sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) -``` - -Because the query output relies on the actual usage of your database, it -may be affected by factors such as: - -- Certain queries never being executed, thus not being able to use certain - indexes. -- Certain tables having little data, resulting in PostgreSQL using sequence - scans instead of index scans. - -This data is only reliable for a frequently used database with -plenty of data, and using as many GitLab features as possible. - -## Requirements for naming indexes - -Indexes with complex definitions must be explicitly named rather than -relying on the implicit naming behavior of migration methods. In short, -that means you **must** provide an explicit name argument for an index -created with one or more of the following options: - -- `where` -- `using` -- `order` -- `length` -- `type` -- `opclass` - -### Considerations for index names - -Check our [Constraints naming conventions](database/constraint_naming_convention.md) page. - -### Why explicit names are required - -As Rails is database agnostic, it generates an index name only -from the required options of all indexes: table name and column names. -For example, imagine the following two indexes are created in a migration: - -```ruby -def up - add_index :my_table, :my_column - - add_index :my_table, :my_column, where: 'my_column IS NOT NULL' -end -``` - -Creation of the second index would fail, because Rails would generate -the same name for both indexes. - -This naming issue is further complicated by the behavior of the `index_exists?` method. -It considers only the table name, column names, and uniqueness specification -of the index when making a comparison. Consider: - -```ruby -def up - unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL') - add_index :my_table, :my_column, where: 'my_column IS NOT NULL' - end -end -``` - -The call to `index_exists?` returns true if **any** index exists on -`:my_table` and `:my_column`, and index creation is bypassed. - -The `add_concurrent_index` helper is a requirement for creating indexes -on populated tables. Because it cannot be used inside a transactional -migration, it has a built-in check that detects if the index already -exists. In the event a match is found, index creation is skipped. -Without an explicit name argument, Rails can return a false positive -for `index_exists?`, causing a required index to not be created -properly. By always requiring a name for certain types of indexes, the -chance of error is greatly reduced. - -## Temporary indexes - -There may be times when an index is only needed temporarily. - -For example, in a migration, a column of a table might be conditionally -updated. To query which columns must be updated in the -[query performance guidelines](query_performance.md), an index is needed -that would otherwise not be used. - -In these cases, consider a temporary index. To specify a -temporary index: - -1. Prefix the index name with `tmp_` and follow the [naming conventions](database/constraint_naming_convention.md). -1. Create a follow-up issue to remove the index in the next (or future) milestone. -1. Add a comment in the migration mentioning the removal issue. - -A temporary migration would look like: - -```ruby -INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled' - -def up - # Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234 - add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME -end - -def down - remove_concurrent_index_by_name :projects, INDEX_NAME -end -``` - -## Create indexes asynchronously - -For very large tables, index creation can be a challenge to manage. -While `add_concurrent_index` creates indexes in a way that does not block -normal traffic, it can still be problematic when index creation runs for -many hours. Necessary database operations like `autovacuum` cannot run, and -on GitLab.com, the deployment process is blocked waiting for index -creation to finish. - -To limit impact on GitLab.com, a process exists to create indexes -asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, -index creation can proceed at a lower level of risk. - -### Schedule index creation for a low-impact time - -1. [Schedule the index to be created](#schedule-the-index-to-be-created). -1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). -1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). - -### Schedule the index to be created - -Create an MR with a post-deployment migration which prepares the index -for asynchronous creation. An example of creating an index using -the asynchronous index helpers can be seen in the block below. This migration -enters the index name and definition into the `postgres_async_indexes` -table. The process that runs on weekends pulls indexes from this -table and attempt to create them. - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -def up - prepare_async_index :ci_builds, :some_column, name: INDEX_NAME -end - -def down - unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME -end -``` - -### Verify the MR was deployed and the index exists in production - -You can verify if the MR was deployed to GitLab.com by executing -`/chatops run auto_deploy status <merge_sha>`. To verify existence of -the index, you can: - -- Use a meta-command in #database-lab, such as: `\d <index_name>`. - - Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). -- Ask someone in #database to check if the index exists. -- With proper access, you can also verify directly on production or in a -production clone. - -### Add a migration to create the index synchronously - -After the index is verified to exist on the production database, create a second -merge request that adds the index synchronously. The schema changes must be -updated and committed to `structure.sql` in this second merge request. -The synchronous migration results in a no-op on GitLab.com, but you should still add the -migration as expected for other installations. The below block -demonstrates how to create the second migration for the previous -asynchronous example. - -**WARNING:** -Verify that the index exists in production before merging a second migration with `add_concurrent_index`. -If the second migration is deployed before the index has been created, -the index is created synchronously when the second migration executes. - -```ruby -# in db/post_migrate/ - -INDEX_NAME = 'index_ci_builds_on_some_column' - -disable_ddl_transaction! - -def up - add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME -end - -def down - remove_concurrent_index_by_name :ci_builds, INDEX_NAME -end -``` - -## Test database index changes locally - -You must test the database index changes locally before creating a merge request. - -### Verify indexes created asynchronously - -Use the asynchronous index helpers on your local environment to test changes for creating an index: - -1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console. -1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table. -1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously. -1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists. +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md index 51c6e86bb49..c00d9da5d16 100644 --- a/doc/development/adding_service_component.md +++ b/doc/development/adding_service_component.md @@ -60,7 +60,7 @@ NOTE: Code shipped with GitLab needs to use a license approved by the Legal team. See the list of [existing approved licenses](https://about.gitlab.com/handbook/engineering/open-source/#using-open-source-libraries). -Notify the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/) when adding a new dependency that must be compiled. We must be able to compile the dependency on all supported platforms. +Notify the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/) when adding a new dependency that must be compiled. We must be able to compile the dependency on all supported platforms. New services to be bundled with GitLab need to be available in the following environments. @@ -83,7 +83,7 @@ In order for a service to be bundled for end-users or GitLab.com, it needs to be Dependencies should be kept up to date and be tracked for security updates. For the Rails codebase, the JavaScript and Ruby dependencies are scanned for vulnerabilities using GitLab [dependency scanning](../user/application_security/dependency_scanning/index.md). -In addition, any system dependencies used in Omnibus packages or the Cloud Native images should be added to the [dependency update automation](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/maintenance/dependencies.io.html#adding-new-dependencies). +In addition, any system dependencies used in Omnibus packages or the Cloud Native images should be added to the [dependency update automation](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/maintenance/dependencies.io.html#adding-new-dependencies). ## Release management diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 37de7044765..0b36b9b2f2f 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -98,11 +98,8 @@ See the [deprecating schema items](#deprecating-schema-items) section for how to ### Breaking change exemptions -Two scenarios exist where schema items are exempt from the deprecation process, -and can be removed or changed at any time without notice. These are schema items that either: - -- Use the [`feature_flag` property](#feature_flag-property) _and_ the flag is disabled by default. -- Are [marked as alpha](#marking-schema-items-as-alpha). +Schema items [marked as alpha](#mark-schema-items-as-alpha) are exempt from the deprecation process, +and can be removed or changed at any time without notice. ## Global IDs @@ -216,7 +213,6 @@ Further reading: - [GraphQL Best Practices Guide](https://graphql.org/learn/best-practices/#nullability). - GraphQL documentation on [Object types and fields](https://graphql.org/learn/schema/#object-types-and-fields). -- [GraphQL Best Practices Guide](https://graphql.org/learn/best-practices/#nullability) - [Using nullability in GraphQL](https://www.apollographql.com/blog/graphql/basics/using-nullability-in-graphql/) ### Exposing Global IDs @@ -249,8 +245,7 @@ end NOTE: For specifics on implementation, see [Pagination implementation](#pagination-implementation). -GraphQL uses [cursor based -pagination](https://graphql.org/learn/pagination/#pagination-and-edges) +GraphQL uses [cursor based pagination](https://graphql.org/learn/pagination/#pagination-and-edges) to expose collections of items. This provides the clients with a lot of flexibility while also allowing the backend to use different pagination models. @@ -472,93 +467,38 @@ end ## Feature flags -Developers can add [feature flags](../development/feature_flags/index.md) to GraphQL -fields in the following ways: - -- Add the [`feature_flag` property](#feature_flag-property) to a field. This allows the field to be _hidden_ - from the GraphQL schema when the flag is disabled. -- [Toggle the return value](#toggle-the-value-of-a-field) when resolving the field. - -You can refer to these guidelines to decide which approach to use: - -- If your field is experimental, and its name or type is subject to - change, use the [`feature_flag` property](#feature_flag-property). -- If your field is stable and its definition doesn't change, even after the flag is - removed, [toggle the return value](#toggle-the-value-of-a-field) of the field instead. Note that - [all fields should be nullable](#nullable-fields) anyway. -- If your field will be accessed from frontend using the `@include` or `@skip` directive, [do not use the `feature_flag` property](#frontend-and-backend-feature-flag-strategies). - -### `feature_flag` property - -The `feature_flag` property allows you to toggle the field's -[visibility](https://graphql-ruby.org/authorization/visibility.html) -in the GraphQL schema. This removes the field from the schema -when the flag is disabled. - -A description is [appended](https://gitlab.com/gitlab-org/gitlab/-/blob/497b556/app/graphql/types/base_field.rb#L44-53) -to the field indicating that it is behind a feature flag. - -WARNING: -If a client queries for the field when the feature flag is disabled, the query -fails. Consider this when toggling the visibility of the feature on or off on -production. - -The `feature_flag` property does not allow the use of -[feature gates based on actors](../development/feature_flags/index.md). -This means that the feature flag cannot be toggled only for particular -projects, groups, or users, but instead can only be toggled globally for -everyone. - -Example: - -```ruby -field :test_field, type: GraphQL::Types::String, - null: true, - description: 'Some test field.', - feature_flag: :my_feature_flag -``` - -### Frontend and Backend feature flag strategies - -#### Directives - -When feature flags are used in the frontend to control the `@include` and `@skip` directives, do not use the `feature_flag` -property on the server-side. For the accepted backend workaround, see [Toggle the value of a field](#toggle-the-value-of-a-field). It is recommended that the feature flag used in this approach be the same for frontend and backend. - -Even if the frontend directives evaluate to `@include:false` / `@skip:true`, the guarded entity is sent to the backend and matched -against the GraphQL schema. We would then get an exception due to a schema mismatch. See the [frontend GraphQL guide](../development/fe_guide/graphql.md#the-include-directive) for more guidance. +You can implement [feature flags](../development/feature_flags/index.md) in GraphQL to toggle: -#### Different versions of a query +- The return value of a field. +- The behavior of an argument or mutation. -See the guide frontend GraphQL guide for [different versions of a query](../development/fe_guide/graphql.md#different-versions-of-a-query), and [why it is not the preferred approach](../development/fe_guide/graphql.md#avoiding-multiple-query-versions) - -### Toggle the value of a field - -This method of using feature flags for fields is to toggle the -return value of the field. This can be done in the resolver, in the +This can be done in a resolver, in the type, or even in a model method, depending on your preference and situation. -Consider also [marking the field as Alpha](#marking-schema-items-as-alpha) -while the value of the field can be toggled. You can -[change or remove Alpha fields at any time](#breaking-change-exemptions) without needing to deprecate them. -This also signals to consumers of the public GraphQL API that the field is not +NOTE: +It's recommended that you also [mark the item as Alpha](#mark-schema-items-as-alpha) while it is behind a feature flag. +This signals to consumers of the public GraphQL API that the field is not meant to be used yet. +You can also +[change or remove Alpha items at any time](#breaking-change-exemptions) without needing to deprecate them. When the flag is removed, "release" +the schema item by removing its Alpha property to make it public. -When applying a feature flag to toggle the value of a field, the -`description` of the field must: +### Descriptions for feature flagged items -- State that the value of the field can be toggled by a feature flag. +When using a feature flag to toggle the value or behavior of a schema item, the +`description` of the item must: + +- State that the value or behavior can be toggled by a feature flag. - Name the feature flag. -- State what the field returns when the feature flag is disabled (or +- State what the field returns, or behavior is, when the feature flag is disabled (or enabled, if more appropriate). -Example: +Example of a feature-flagged field: ```ruby -field :foo, GraphQL::Types::String, - null: true, - deprecated: { reason: :alpha, milestone: '10.0' }, +field :foo, GraphQL::Types::String, null: true, + alpha: { milestone: '10.0' }, description: 'Some test field. Returns `null`' \ 'if `my_feature_flag` feature flag is disabled.' @@ -567,6 +507,26 @@ def foo end ``` +Example of a feature-flagged argument: + +```ruby +argument :foo, type: GraphQL::Types::String, required: false, + alpha: { milestone: '10.0' }, + description: 'Some test argument. Is ignored if ' \ + '`my_feature_flag` feature flag is disabled.' + +def resolve(args) + args.delete(:foo) unless Feature.enabled?(:my_feature_flag, object) + # ... +end +``` + +### `feature_flag` property (deprecated) + +NOTE: +This property is deprecated and should no longer be used. The property +has been temporarily renamed to `_deprecated_feature_flag` and support for it will be removed in [#369202](https://gitlab.com/gitlab-org/gitlab/-/issues/369202). + ## Deprecating schema items The GitLab GraphQL API is versionless, which means we maintain backwards @@ -586,6 +546,7 @@ To deprecate a schema item in GraphQL: See also: - [Aliasing and deprecating mutations](#aliasing-and-deprecating-mutations). +- [Marking schema items as Alpha](#mark-schema-items-as-alpha). - [How to filter Kibana for queries that used deprecated fields](graphql_guide/monitoring.md#queries-that-used-a-deprecated-field). ### Create a deprecation issue @@ -715,7 +676,7 @@ To allow clients to continue to interact with the mutation unchanged, edit the ` ```ruby DEPRECATIONS = [ - Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.0') + Gitlab::Graphql::DeprecationsBase::NameDeprecation.new(old_name: 'PrometheusService', new_name: 'Integrations::Prometheus', milestone: '14.0') ].freeze ``` @@ -745,28 +706,37 @@ aware of the support. The documentation will mention that the old Global ID style is now deprecated. -## Marking schema items as Alpha +## Mark schema items as Alpha -Fields, arguments, enum values, and mutations can be marked as being in -[alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga). +You can mark GraphQL schema items (fields, arguments, enum values, and mutations) as +[Alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga). -An item marked as "alpha" is exempt from the deprecation process and can be removed -at any time without notice. +An item marked as Alpha is [exempt from the deprecation process](#breaking-change-exemptions) and can be removed +at any time without notice. Mark an item as Alpha when it is +subject to change and not ready for public use. -This leverages GraphQL deprecations to cause the schema item to appear as deprecated, -and will be described as being in "alpha" in our generated docs and its GraphQL description. +NOTE: +Only mark new items as Alpha. Never mark existing items +as Alpha because they're already public. -To mark a schema item as being in "alpha", use the `deprecated:` keyword with `reason: :alpha`. -You must provide the `milestone:` that introduced the alpha item. +To mark a schema item as Alpha, use the `alpha:` keyword. +You must provide the `milestone:` that introduced the Alpha item. For example: ```ruby field :token, GraphQL::Types::String, null: true, - deprecated: { reason: :alpha, milestone: '10.0' }, + alpha: { milestone: '10.0' }, description: 'Token for login.' ``` +Alpha GraphQL items is a custom GitLab feature that leverages GraphQL deprecations. An Alpha item +appears as deprecated in the GraphQL schema. Like all deprecated schema items, you can test an +Alpha field in [GraphiQL](../api/graphql/index.md#graphiql). However, be aware that the GraphiQL +autocomplete editor doesn't suggest deprecated fields. + +The item shows as Alpha in our generated GraphQL documentation and its GraphQL schema description. + ## Enums GitLab GraphQL enums are defined in `app/graphql/types`. When defining new enums, the @@ -816,7 +786,7 @@ Enum values can be deprecated using the ### Defining GraphQL enums dynamically from Rails enums -If your GraphQL enum is backed by a [Rails enum](creating_enums.md), then consider +If your GraphQL enum is backed by a [Rails enum](database/creating_enums.md), then consider using the Rails enum to dynamically define the GraphQL enum values. Doing so binds the GraphQL enum values to the Rails enum definition, so if values are ever added to the Rails enum then the GraphQL enum automatically reflects the change. @@ -1641,8 +1611,8 @@ correctly rendered to the clients. ### Errors in mutations -We encourage following the practice of [errors as -data](https://graphql-ruby.org/mutations/mutation_errors) for mutations, which +We encourage following the practice of +[errors as data](https://graphql-ruby.org/mutations/mutation_errors) for mutations, which distinguishes errors by who they are relevant to, defined by who can deal with them. diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index df2f3c337cd..b72ef1bffc4 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -59,7 +59,7 @@ end ## Declared parameters -> Grape allows you to access only the parameters that have been declared by your +Grape allows you to access only the parameters that have been declared by your `params` block. It filters out the parameters that have been passed, but are not allowed. @@ -67,7 +67,7 @@ allowed. ### Exclude parameters from parent namespaces -> By default `declared(params)`includes parameters that were defined in all +By default `declared(params)`includes parameters that were defined in all parent namespaces. – <https://github.com/ruby-grape/grape#include-parent-namespaces> @@ -110,15 +110,15 @@ Model.create(foo: params[:foo]) With Grape v1.3+, Array types must be defined with a `coerce_with` block, or parameters, fails to validate when passed a string from an -API request. See the [Grape upgrading -documentation](https://github.com/ruby-grape/grape/blob/master/UPGRADING.md#ensure-that-array-types-have-explicit-coercions) +API request. See the +[Grape upgrading documentation](https://github.com/ruby-grape/grape/blob/master/UPGRADING.md#ensure-that-array-types-have-explicit-coercions) for more details. ### Automatic coercion of nil inputs Prior to Grape v1.3.3, Array parameters with `nil` values would -automatically be coerced to an empty Array. However, due to [this pull -request in v1.3.3](https://github.com/ruby-grape/grape/pull/2040), this +automatically be coerced to an empty Array. However, due to +[this pull request in v1.3.3](https://github.com/ruby-grape/grape/pull/2040), this is no longer the case. For example, suppose you define a PUT `/test` request that has an optional parameter: @@ -259,8 +259,8 @@ In situations where the same model has multiple entities in the API discretion with applying this scope. It may be that you optimize for the most basic entity, with successive entities building upon that scope. -The `with_api_entity_associations` scope also [automatically preloads -data](https://gitlab.com/gitlab-org/gitlab/-/blob/19f74903240e209736c7668132e6a5a735954e7c/app%2Fmodels%2Ftodo.rb#L34) +The `with_api_entity_associations` scope also +[automatically preloads data](https://gitlab.com/gitlab-org/gitlab/-/blob/19f74903240e209736c7668132e6a5a735954e7c/app%2Fmodels%2Ftodo.rb#L34) for `Todo` _targets_ when returned in the [to-dos API](../api/todos.md). For more context and discussion about preloading see diff --git a/doc/development/application_limits.md b/doc/development/application_limits.md index 2826b8a3bc4..ceb3c124d1a 100644 --- a/doc/development/application_limits.md +++ b/doc/development/application_limits.md @@ -15,8 +15,7 @@ First of all, you have to gather information and decide which are the different limits that are set for the different GitLab tiers. Coordinate with others to [document](../administration/instance_limits.md) and communicate those limits. -There is a guide about [introducing application -limits](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits). +There is a guide about [introducing application limits](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits). ## Implement plan limits diff --git a/doc/development/application_slis/index.md b/doc/development/application_slis/index.md index 8d7941865e1..27e69ff3445 100644 --- a/doc/development/application_slis/index.md +++ b/doc/development/application_slis/index.md @@ -8,16 +8,14 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/525) in GitLab 14.4 -It is possible to define [Service Level Indicators -(SLIs)](https://en.wikipedia.org/wiki/Service_level_indicator) +It is possible to define [Service Level Indicators(SLIs)](https://en.wikipedia.org/wiki/Service_level_indicator) directly in the Ruby codebase. This keeps the definition of operations and their success close to the implementation and allows the people building features to easily define how these features should be monitored. Defining an SLI causes 2 -[Prometheus -counters](https://prometheus.io/docs/concepts/metric_types/#counter) +[Prometheus counters](https://prometheus.io/docs/concepts/metric_types/#counter) to be emitted from the rails application: - `gitlab_sli:<sli name>:total`: incremented for each operation. @@ -47,10 +45,9 @@ for clarity, they define different metric names: As shown in this example, they can share a base name (`foo` in this example). We recommend this when they refer to the same operation. -Before the first scrape, it is important to have [initialized the SLI -with all possible -label-combinations](https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics). This -avoid confusing results when using these counters in calculations. +Before the first scrape, it is important to have +[initialized the SLI with all possible label-combinations](https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics). +This avoid confusing results when using these counters in calculations. To initialize an SLI, use the `.initialize_sli` class method, for example: diff --git a/doc/development/application_slis/rails_request_apdex.md b/doc/development/application_slis/rails_request_apdex.md index 3e3cd100183..033bffbf608 100644 --- a/doc/development/application_slis/rails_request_apdex.md +++ b/doc/development/application_slis/rails_request_apdex.md @@ -231,8 +231,7 @@ end ### Error budget attribution and ownership This SLI is used for service level monitoring. It feeds into the -[error budget for stage -groups](../stage_group_observability/index.md#error-budget). For this +[error budget for stage groups](../stage_group_observability/index.md#error-budget). For this particular SLI, we have opted everyone out by default to give time to set the correct urgencies on endpoints before it affects a group's error budget. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index a61a891b096..10d6c0ae9c9 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -89,7 +89,7 @@ new features and services must be written to consider Kubernetes compatibility * The simplest way to ensure this, is to add support for your feature or service to [the official GitLab Helm chart](https://docs.gitlab.com/charts/) or reach out to -[the Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/#how-to-work-with-distribution). +[the Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/#how-to-work-with-distribution). Refer to the [process for adding new service components](adding_service_component.md) for more details. diff --git a/doc/development/audit_event_guide/index.md b/doc/development/audit_event_guide/index.md index 14cd2fd1dc3..0c66189a6f6 100644 --- a/doc/development/audit_event_guide/index.md +++ b/doc/development/audit_event_guide/index.md @@ -29,13 +29,13 @@ If you have any questions, please reach out to `@gitlab-org/manage/compliance` t To instrument an audit event, the following attributes should be provided: -| Attribute | Type | Required? | Description | -|:-------------|:---------------------|:----------|:-----------------------------------------------------------------| -| `name` | String | false | Action name to be audited. Used for error tracking | -| `author` | User | true | User who authors the change | -| `scope` | User, Project, Group | true | Scope which the audit event belongs to | -| `target` | Object | true | Target object being audited | -| `message` | String | true | Message describing the action | +| Attribute | Type | Required? | Description | +|:-------------|:---------------------|:----------|:------------------------------------------------------------------| +| `name` | String | false | Action name to be audited. Used for error tracking | +| `author` | User | true | User who authors the change | +| `scope` | User, Project, Group | true | Scope which the audit event belongs to | +| `target` | Object | true | Target object being audited | +| `message` | String | true | Message describing the action ([not translated](#i18n-and-the-audit-event-message-attribute)) | | `created_at` | DateTime | false | The time when the action occurred. Defaults to `DateTime.current` | ## How to instrument new Audit Events @@ -195,7 +195,7 @@ deactivate B ``` In addition to recording to the database, we also write these events to -[a log file](../../administration/logs.md#audit_jsonlog). +[a log file](../../administration/logs/index.md#audit_jsonlog). ## Event streaming @@ -210,3 +210,10 @@ a large amount of data. See [this merge request](https://gitlab.com/gitlab-org/g for an example. This feature is under heavy development. Follow the [parent epic](https://gitlab.com/groups/gitlab-org/-/epics/5925) for updates on feature development. + +### I18N and the Audit Event `:message` attribute + +We intentionally do not translate audit event messages because translated messages would be saved in the database and served to users, regardless of their locale settings. + +This could mean, for example, that we use the locale for the currently logged in user to record an audit event message and stream the message to an external streaming +destination in the wrong language for that destination. Users could find that confusing. diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index 2989e10a124..55ab234cc68 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -20,8 +20,8 @@ based on your project contents. When Auto DevOps is enabled for a project, the user does not need to explicitly include any pipeline configuration through a [`.gitlab-ci.yml` file](../ci/yaml/index.md). -In the absence of a `.gitlab-ci.yml` file, the [Auto DevOps CI -template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) +In the absence of a `.gitlab-ci.yml` file, the +[Auto DevOps CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) is used implicitly to configure the pipeline for the project. This template is a top-level template that includes other sub-templates, which then defines jobs. diff --git a/doc/development/backend/create_source_code_be/index.md b/doc/development/backend/create_source_code_be/index.md index ad4e25dc815..e1ee78731de 100644 --- a/doc/development/backend/create_source_code_be/index.md +++ b/doc/development/backend/create_source_code_be/index.md @@ -13,7 +13,7 @@ of the [Create stage](https://about.gitlab.com/handbook/product/categories/#crea of the [DevOps lifecycle](https://about.gitlab.com/handbook/product/categories/#devops-stages). We interface with the Gitaly and Code Review teams, and work closely with the -[Create:Source Code Frontend team](https://about.gitlab.com/handbook/engineering/development/dev/create-source-code-fe). The features +[Create:Source Code Frontend team](https://about.gitlab.com/handbook/engineering/development/dev/create/create-source-code-fe/). The features we work with are listed on the [Features by Group Page](https://about.gitlab.com/handbook/product/categories/features/#createsource-code-group). diff --git a/doc/development/backend/ruby_style_guide.md b/doc/development/backend/ruby_style_guide.md index c86f21d4bac..a9fee02a15a 100644 --- a/doc/development/backend/ruby_style_guide.md +++ b/doc/development/backend/ruby_style_guide.md @@ -16,6 +16,8 @@ document the new rule. For every new guideline, add it in a new section and link [version history note](../documentation/versions.md#add-a-version-history-item) to provide context and serve as a reference. +See also [guidelines for reusing abstractions](../reusing_abstractions.md). + Everything listed here can be [reopened for discussion](https://about.gitlab.com/handbook/values/#disagree-commit-and-disagree). ## Instance variable access using `attr_reader` diff --git a/doc/development/build_test_package.md b/doc/development/build_test_package.md index 89b13efc1aa..4645bd02d9e 100644 --- a/doc/development/build_test_package.md +++ b/doc/development/build_test_package.md @@ -13,8 +13,8 @@ pipeline that can be used to trigger a pipeline in the Omnibus GitLab repository that will create: - A deb package for Ubuntu 16.04, available as a build artifact, and -- A Docker image, which is pushed to the [Omnibus GitLab container - registry](https://gitlab.com/gitlab-org/omnibus-gitlab/container_registry) +- A Docker image, which is pushed to the + [Omnibus GitLab container registry](https://gitlab.com/gitlab-org/omnibus-gitlab/container_registry) (images titled `gitlab-ce` and `gitlab-ee` respectively and image tag is the commit which triggered the pipeline). diff --git a/doc/development/cached_queries.md b/doc/development/cached_queries.md index b0bf7c7b6f5..7af4c302e93 100644 --- a/doc/development/cached_queries.md +++ b/doc/development/cached_queries.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Memory +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/caching.md b/doc/development/caching.md index 7c51bd595f7..5ae6484436e 100644 --- a/doc/development/caching.md +++ b/doc/development/caching.md @@ -118,8 +118,8 @@ Is the cache being added "worthy"? This can be hard to measure, but you can cons - `tail -f log/development.log | grep "Rendered "` - After you're looking in the right place: - Remove or comment out sections of code until you find the cause. - - Use `binding.pry` to poke about in live requests. This requires a foreground - web process like [Thin](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/pry.md). + - Use `binding.pry` to poke about in live requests. This requires a + [foreground web process](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/pry.md). #### Verification @@ -301,10 +301,10 @@ it's time to look at a custom solution: In short: the oldest stuff is replaced with new stuff: -- A [useful article](https://redis.io/topics/lru-cache) about configuring Redis as an LRU cache. +- A [useful article](https://redis.io/docs/manual/eviction/) about configuring Redis as an LRU cache. - Lots of options for different cache eviction strategies. - You probably want `allkeys-lru`, which is functionally similar to Memcached. -- In Redis 4.0 and later, [allkeys-lfu is available](https://redis.io/topics/lru-cache#the-new-lfu-mode), +- In Redis 4.0 and later, [allkeys-lfu is available](https://redis.io/docs/manual/eviction/#the-new-lfu-mode), which is similar but different. - We handle all explicit deletes using UNLINK instead of DEL now, which allows Redis to reclaim memory in its own time, rather than immediately. diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 83919bab671..c5b234069e3 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -190,8 +190,8 @@ editor. Once closed, Git presents you with a new text editor instance to edit the commit message of commit B. Add the trailer, then save and quit the editor. If all went well, commit B is now updated. -For more information about interactive rebases, take a look at [the Git -documentation](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History). +For more information about interactive rebases, take a look at +[the Git documentation](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History). --- diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index 2065021c61b..7309b92c702 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -59,5 +59,5 @@ To request access to ChatOps on GitLab.com: ## See also - [ChatOps Usage](../ci/chatops/index.md) -- [Understanding EXPLAIN plans](understanding_explain_plans.md) +- [Understanding EXPLAIN plans](database/understanding_explain_plans.md) - [Feature Groups](feature_flags/index.md#feature-groups) diff --git a/doc/development/code_intelligence/index.md b/doc/development/code_intelligence/index.md index 3a8845084c3..a89730383e4 100644 --- a/doc/development/code_intelligence/index.md +++ b/doc/development/code_intelligence/index.md @@ -35,8 +35,8 @@ sequenceDiagram Workhorse-->>-Runner: request results ``` -1. The CI/CD job generates a document in an LSIF format (usually `dump.lsif`) using [an - indexer](https://lsif.dev) for the language of a project. The format +1. The CI/CD job generates a document in an LSIF format (usually `dump.lsif`) using + [an indexer](https://lsif.dev) for the language of a project. The format [describes](https://github.com/sourcegraph/sourcegraph/blob/main/doc/code_intelligence/explanations/writing_an_indexer.md) interactions between a method or function and its definitions or references. The document is marked to be stored as an LSIF report artifact. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 1225260e600..e9e546c6f9b 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -62,6 +62,9 @@ Team members' domain expertise can be viewed on the [engineering projects](https ### Reviewer roulette +NOTE: +Reviewer roulette is an internal tool for use on GitLab.com, and not available for use on customer installations. + The [Danger bot](dangerbot.md) randomly picks a reviewer and a maintainer for each area of the codebase that your merge request seems to touch. It only makes **recommendations** and you should override it if you think someone else is a better @@ -110,6 +113,12 @@ The [Roulette dashboard](https://gitlab-org.gitlab.io/gitlab-roulette) contains: For more information, review [the roulette README](https://gitlab.com/gitlab-org/gitlab-roulette). +As an experiment, we want to introduce a `local` reviewer status for database reviews. Local reviewers are reviewers +focusing on work from a team/stage, but not outside of it. This helps to focus and build great domain +knowledge. We are not introducing changes to the reviewer roulette till we evaluate the impact and feedback from this +experiment. We ask to respect reviewers who decline reviews based on their focus on `local` reviews. For tracking purposes, +please use in your personal YAML file entry: `- reviewer database local` instead of `- reviewer database`. + ### Approval guidelines As described in the section on the responsibility of the maintainer below, you @@ -135,7 +144,7 @@ with [domain expertise](#domain-experts). More information about license compatibility can be found in our [GitLab Licensing and Compatibility documentation](licensing.md). 1. If your merge request includes a new dependency or a file system change, it must be - **approved by a [Distribution team member](https://about.gitlab.com/company/team/)**. See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/#how-to-work-with-distribution) for more details. + **approved by a [Distribution team member](https://about.gitlab.com/company/team/)**. See how to work with the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/#how-to-work-with-distribution) for more details. 1. If your merge request includes documentation changes, it must be **approved by a [Technical writer](https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments)**, based on assignments in the appropriate [DevOps stage group](https://about.gitlab.com/handbook/product/categories/#devops-stages). @@ -144,7 +153,7 @@ with [domain expertise](#domain-experts). by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. 1. If your merge request only includes end-to-end changes (*4*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** 1. If your merge request includes a new or updated [application limit](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. -1. If your merge request includes Product Intelligence (telemetry or analytics) changes, it should be reviewed and approved by a [Product Intelligence engineer](https://gitlab.com/gitlab-org/growth/product-intelligence/engineers). +1. If your merge request includes Product Intelligence (telemetry or analytics) changes, it should be reviewed and approved by a [Product Intelligence engineer](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/engineers). 1. If your merge request includes an addition of, or changes to a [Feature spec](testing_guide/testing_levels.md#frontend-feature-tests), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa) or [Quality reviewer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_qa)**. 1. If your merge request introduces a new service to GitLab (Puma, Sidekiq, Gitaly are examples), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. See the [process for adding a service component to GitLab](adding_service_component.md) for details. 1. If your merge request includes changes related to authentication or authorization, it must be **approved by a [Manage:Authentication and Authorization team member](https://about.gitlab.com/company/team/)**. Check the [code review section on the group page](https://about.gitlab.com/handbook/engineering/development/dev/manage/authentication-and-authorization/#additional-considerations) for more details. Patterns for files known to require review from the team are listed in the in the `Authentication and Authorization` section of the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS) file, and the team will be listed in the approvers section of all merge requests that modify these files. @@ -176,7 +185,7 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering 1. I have tested this MR in [all supported browsers](../install/requirements.md#supported-web-browsers), or determined that this testing is not needed. 1. I have confirmed that this change is [backwards compatible across updates](multi_version_compatibility.md), or I have decided that this does not apply. 1. I have properly separated EE content from FOSS, or this MR is FOSS only. - - [Where should EE code go?](ee_features.md#separation-of-ee-code) + - [Where should EE code go?](ee_features.md) 1. I have considered that existing data may be surprisingly varied. For example, a new model validation can break existing records. Consider making validation on existing data optional rather than required if you haven't confirmed that existing data will pass validation. ##### Performance, reliability, and availability @@ -278,13 +287,29 @@ This saves reviewers time and helps authors catch mistakes earlier. ### The responsibility of the reviewer +Reviewers are responsible for reviewing the specifics of the chosen solution. + [Review the merge request](#reviewing-a-merge-request) thoroughly. Verify that the merge request meets all [contribution acceptance criteria](contributing/merge_request_workflow.md#contribution-acceptance-criteria). -If a merge request is too large, fixes more than one issue, or implements more -than one feature, you should guide the author towards splitting the merge request -into smaller merge requests. +Some merge requests may require domain experts to help with the specifics. +Reviewers, if they are not a domain expert in the area, can do any of the following: + +- Review the merge request and loop in a domain expert for another review. This expert + can either be another reviewer or a maintainer. +- Pass the review to another reviewer they deem more suitable. +- If no domain experts are available, review on a best-effort basis. + +You should guide the author towards splitting the merge request into smaller merge requests if it is: + +- Too large. +- Fixes more than one issue. +- Implements more than one feature. +- Has a high complexity resulting in additional risk. + +The author may choose to request that the current maintainers and reviewers review the split MRs +or request a new group of maintainers and reviewers. When you are confident that it meets all requirements, you should: @@ -308,19 +333,6 @@ Because a maintainer's job only depends on their knowledge of the overall GitLab codebase, and not that of any specific domain, they can review, approve, and merge merge requests from any team and in any product area. -A maintainer should ask the author to make a merge request smaller if it is: - -- Too large. -- Fixes more than one issue. -- Implements more than one feature. -- Has a high complexity resulting in additional risk. - -The maintainer, any of the -reviewers, or a merge request coach can step up to help the author to divide work -into smaller iterations, and guide the author on how to split the merge request. -The author may choose to request that the current maintainers and reviewers review the split MRs -or request a new group of maintainers and reviewers. - Maintainers do their best to also review the specifics of the chosen solution before merging, but as they are not necessarily [domain experts](#domain-experts), they may be poorly placed to do so without an unreasonable investment of time. In those cases, they @@ -401,7 +413,7 @@ first time. of your shiny new branch, read through the entire diff. Does it make sense? Did you include something unrelated to the overall purpose of the changes? Did you forget to remove any debugging code? -- Write a detailed description as outlined in the [merge request guidelines](contributing/merge_request_workflow.md#merge-request-guidelines). +- Write a detailed description as outlined in the [merge request guidelines](contributing/merge_request_workflow.md#merge-request-guidelines-for-contributors). Some reviewers may not be familiar with the product feature or area of the codebase. Thorough descriptions help all reviewers understand your request and test effectively. @@ -445,7 +457,7 @@ You can also use `workflow::ready for review` label. That means that your merge When your merge request receives an approval from the first reviewer it can be passed to a maintainer. You should default to choosing a maintainer with [domain expertise](#domain-experts), and otherwise follow the Reviewer Roulette recommendation or use the label `ready for merge`. -Sometimes, a maintainer may not be available for review. They could be out of the office or [at capacity](#review-response-slo). +Sometimes, a maintainer may not be available for review. They could be out of the office or [at capacity](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo). You can and should check the maintainer's availability in their profile. If the maintainer recommended by the roulette is not available, choose someone else from that list. @@ -466,6 +478,8 @@ experience, refactors the existing code). Then: - Offer alternative implementations, but assume the author already considered them. ("What do you think about using a custom validator here?") - Seek to understand the author's perspective. +- Check out the branch, and test the changes locally. You can decide how much manual testing you want to perform. + Your testing might result in opportunities to add automated tests. - If you don't understand a piece of code, _say so_. There's a good chance someone else would be confused by it as well. - Ensure the author is clear on what is required from them to address/resolve the suggestion. @@ -494,34 +508,29 @@ Before taking the decision to merge: - If the MR contains both Quality and non-Quality-related changes, the MR should be merged by the relevant maintainer for user-facing changes (backend, frontend, or database) after the Quality related changes are approved by a Software Engineer in Test. If a merge request is fundamentally ready, but needs only trivial fixes (such as -typos), consider demonstrating a [bias for -action](https://about.gitlab.com/handbook/values/#bias-for-action) by making -those changes directly without going back to the author. You can do this by +typos), consider demonstrating a [bias for action](https://about.gitlab.com/handbook/values/#bias-for-action) +by making those changes directly without going back to the author. You can do this by using the [suggest changes](../user/project/merge_requests/reviews/suggestions.md) feature to apply your own suggestions to the merge request. Note that: - If the changes are not straightforward, please prefer allowing the author to make the change. - **Before applying suggestions**, edit the merge request to make sure - [squash and - merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) + [squash and merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) is enabled, otherwise, the pipeline's Danger job fails. - If a merge request does not have squash and merge enabled, and it has more than one commit, then see the note below about rewriting commit history. -As a maintainer, if a merge request that you authored has received all required approvals, it is acceptable to show a [bias for action](https://about.gitlab.com/handbook/values/#bias-for-action) and merge your own MR, if: - -- The last maintainer to review intended to start the merge and did not, OR -- The last maintainer to review started the merge, but some trivial chore caused the pipeline to break. For example, the MR might need a rebase first because of unrelated pipeline issues, or some files might need to be regenerated (like `gitlab.pot`). - - "Trivial" is a subjective measure but we expect project maintainers to exercise their judgement carefully and cautiously. +Authors are not authorized to merge their own merge requests and need to seek another maintainer to merge. +This policy is in place to satisfy the CHG-04 control of the GitLab +[Change Management Controls](https://about.gitlab.com/handbook/engineering/security/security-assurance/security-compliance/guidance/change-management.html). When ready to merge: WARNING: **If the merge request is from a fork, also check the [additional guidelines for community contributions](#community-contributions).** -- Consider using the [Squash and - merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) +- Consider using the [Squash and merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) feature when the merge request has a lot of commits. When merging code, a maintainer should only use the squash feature if the author has already set this option, or if the merge request clearly contains a @@ -541,8 +550,7 @@ WARNING: enough to `main`. - When you set the MR to "Merge When Pipeline Succeeds", you should take over subsequent revisions for anything that would be spotted after that. -- For merge requests that have had [Squash and - merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) set, +- For merge requests that have had [Squash and merge](../user/project/merge_requests/squash_and_merge.md#squash-and-merge) set, the squashed commit's default commit message is taken from the merge request title. You're encouraged to [select a commit with a more informative commit message](../user/project/merge_requests/squash_and_merge.md) before merging. @@ -586,7 +594,7 @@ If the MR source branch is more than 1,000 commits behind the target branch: When an MR needs further changes but the author is not responding for a long period of time, or is unable to finish the MR, GitLab can take it over in accordance with our -[Closing policy for issues and merge requests](contributing/#closing-policy-for-issues-and-merge-requests). +[Closing policy for issues and merge requests](contributing/index.md#closing-policy-for-issues-and-merge-requests). A GitLab engineer (generally the merge request coach) will: 1. Add a comment to their MR saying you'll take it over to be able to get it merged. @@ -683,42 +691,6 @@ Enterprise Edition instance. This has some implications: Ensure that we support object storage for any file storage we need to perform. For more information, see the [uploads documentation](uploads/index.md). -### Review turnaround time - -Because [unblocking others is always a top priority](https://about.gitlab.com/handbook/values/#global-optimization), -reviewers are expected to review merge requests in a timely manner, -even when this may negatively impact their other tasks and priorities. - -Doing so allows everyone involved in the merge request to iterate faster as the -context is fresh in memory, and improves contributors' experience significantly. - -#### Review-response SLO - -To ensure swift feedback to ready-to-review code, we maintain a `Review-response` Service-level Objective (SLO). The SLO is defined as: - -> Review-response SLO = (time when first review is provided) - (time MR is assigned to reviewer) < 2 business days - -If you don't think you can review a merge request in the `Review-response` SLO -time frame, let the author know as soon as possible in the comments -(no later than 36 hours after first receiving the review request) -and try to help them find another reviewer or maintainer who is able to, so that they can be unblocked -and get on with their work quickly. Remove yourself as a reviewer. - -If you think you are at capacity and are unable to accept any more reviews until -some have been completed, communicate this through your GitLab status by setting -the 🔴 `:red_circle:` emoji and mentioning that you are at capacity in the status -text. This guides contributors to pick a different reviewer, helping us to -meet the SLO. - -Of course, if you are out of office and have -[communicated](https://about.gitlab.com/handbook/paid-time-off/#communicating-your-time-off) -this through your GitLab.com Status, authors are expected to realize this and -find a different reviewer themselves. - -When a merge request author has been blocked for longer than -the `Review-response` SLO, they are free to remind the reviewer through Slack or add -another reviewer. - ### Customer critical merge requests A merge request may benefit from being considered a customer critical priority because there is a significant benefit to the business in doing so. diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index ce013a9254b..04915985dc8 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -68,9 +68,9 @@ Check states using your browser's _styles inspector_ to toggle CSS pseudo-classe like `:hover` and others ([Chrome](https://developer.chrome.com/docs/devtools/css/reference/#pseudo-class), [Firefox](https://firefox-source-docs.mozilla.org/devtools-user/page_inspector/how_to/examine_and_edit_css/index.html#viewing-common-pseudo-classes)). -- Account for all applicable states ([error](https://design.gitlab.com/content/error-messages), +- Account for all applicable states ([error](https://design.gitlab.com/content/error-messages/), rest, loading, focus, hover, selected, disabled). -- Account for states dependent on data size ([empty](https://design.gitlab.com/regions/empty-states), +- Account for states dependent on data size ([empty](https://design.gitlab.com/regions/empty-states/), some data, and lots of data). - Account for states dependent on user role, user preferences, and subscription. - Consider animations and transitions, and follow their [guidelines](https://design.gitlab.com/product-foundations/motion/). @@ -113,7 +113,7 @@ When the design is ready, _before_ starting its implementation: At any moment, but usually _during_ or _after_ the design's implementation: -- Contribute [issues to Pajamas](https://design.gitlab.com/get-started/contribute#contribute-an-issue) +- Contribute [issues to Pajamas](https://design.gitlab.com/get-started/contribute/#contribute-an-issue) for additions or enhancements to the design system. - Create issues with the [`~UX debt`](issue_workflow.md#technical-and-ux-debt) label for intentional deviations from the agreed-upon UX requirements due to diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 12fd7c3dc12..6999ffe810e 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -186,21 +186,11 @@ reasons for including it. Mention a maintainer in merge requests that contain: - More than 500 changes. -- Any major [breaking changes](#breaking-changes). +- Any major [breaking changes](../deprecation_guidelines/index.md). - External libraries. If you are not sure who to mention, the reviewer will do this for you early in the merge request process. -#### Breaking changes - -A "breaking change" is any change that requires users to make a corresponding change to their code, settings, or workflow. "Users" might be humans, API clients, or even code classes that "use" another class. Examples of breaking changes include: - -- Removing a user-facing feature without a replacement/workaround. -- Changing the definition of an existing API (by doing things like re-naming query parameters or changing routes). -- Removing a public method from a code class. - -A breaking change can be considered "major" if it affects many users, or represents a significant change in behavior. - #### Issues workflow This [documentation](issue_workflow.md) outlines the current issue workflow: @@ -218,7 +208,7 @@ This [documentation](issue_workflow.md) outlines the current issue workflow: This [documentation](merge_request_workflow.md) outlines the current merge request process. -- [Merge request guidelines](merge_request_workflow.md#merge-request-guidelines) +- [Merge request guidelines](merge_request_workflow.md#merge-request-guidelines-for-contributors) - [Contribution acceptance criteria](merge_request_workflow.md#contribution-acceptance-criteria) - [Definition of done](merge_request_workflow.md#definition-of-done) - [Dependencies](merge_request_workflow.md#dependencies) diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index c6d977cf5ad..d557319b41f 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -53,7 +53,7 @@ Most issues will have labels for at least one of the following: - Priority: `~"priority::1"`, `~"priority::2"`, `~"priority::3"`, `~"priority::4"` - Severity: ~`"severity::1"`, `~"severity::2"`, `~"severity::3"`, `~"severity::4"` -Please add `~"breaking change"` label if the issue can be considered as a [breaking change](index.md#breaking-changes). +Please add `~"breaking change"` label if the issue can be considered as a [breaking change](../deprecation_guidelines/index.md). Please add `~security` label if the issue is related to application security. @@ -161,7 +161,7 @@ For instance, the "DevOps Reports" category is represented by the `devops_reports.name` value is "DevOps Reports". If a category's label doesn't respect this naming convention, it should be specified -with [the `label` attribute](https://about.gitlab.com/handbook/marketing/inbound-marketing/digital-experience/website/#category-attributes) +with [the `label` attribute](https://about.gitlab.com/handbook/marketing/digital-experience/website/#category-attributes) in <https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/categories.yml>. ### Feature labels diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index eff1d2e671d..faa1642d50a 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -33,7 +33,7 @@ some potentially easy issues. To start developing GitLab, download the [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit) and see the [Development section](../../index.md) for the required guidelines. -## Merge request guidelines +## Merge request guidelines for contributors If you find an issue, please submit a merge request with a fix or improvement, if you can, and include tests. If you don't know how to fix the issue but can write a test @@ -45,20 +45,10 @@ request is as follows: 1. [Fork](../../user/project/repository/forking_workflow.md) the project into your personal namespace (or group) on GitLab.com. 1. Create a feature branch in your fork (don't work off your [default branch](../../user/project/repository/branches/default.md)). -1. Write [tests](../rake_tasks.md#run-tests) and code. -1. [Ensure a changelog is created](../changelog.md). -1. If you are writing documentation, make sure to follow the - [documentation guidelines](../documentation/index.md). 1. Follow the [commit messages guidelines](#commit-messages-guidelines). -1. If you have multiple commits, combine them into a few logically organized - commits by [squashing them](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#_squashing), - but do not change the commit history if you're working on shared branches though. +1. If you have multiple commits, combine them into a few logically organized commits. 1. Push the commits to your working branch in your fork. -1. Submit a merge request (MR) to the `main` branch in the main GitLab project. - 1. Your merge request needs at least 1 approval, but depending on your changes - you might need additional approvals. Refer to the [Approval guidelines](../code_review.md#approval-guidelines). - 1. You don't have to select any specific approvers, but you can if you really want - specific people to approve your merge request. +1. Submit a merge request (MR) against the default branch of the upstream project. 1. The MR title should describe the change you want to make. 1. The MR description should give a reason for your change. 1. If you are contributing code, fill in the description according to the default @@ -68,58 +58,15 @@ request is as follows: 1. Use the syntax `Solves #XXX`, `Closes #XXX`, or `Refs #XXX` to mention the issues your merge request addresses. Referenced issues do not [close automatically](../../user/project/issues/managing_issues.md#closing-issues-automatically). You must close them manually once the merge request is merged. - 1. The MR must include *Before* and *After* screenshots if UI changes are made. - 1. Include any steps or setup required to ensure reviewers can view the changes you've made (for example, include any information about feature flags). 1. If you're allowed to, set a relevant milestone and [labels](issue_workflow.md). MR labels should generally match the corresponding issue (if there is one). The group label should reflect the group that executed or coached the work, not necessarily the group that owns the feature. -1. UI changes should use available components from the GitLab Design System, - [Pajamas](https://design.gitlab.com/). -1. If the MR changes CSS classes, please include the list of affected pages, which - can be found by running `grep css-class ./app -R`. -1. If your MR touches code that executes shell commands, reads or opens files, or - handles paths to files on disk, make sure it adheres to the - [shell command guidelines](../shell_commands.md) -1. [Code changes should include observability instrumentation](../code_review.md#observability-instrumentation). -1. If your code needs to handle file storage, see the [uploads documentation](../uploads/index.md). -1. If your merge request adds one or more migrations, make sure to execute all - migrations on a fresh database before the MR is reviewed. If the review leads - to large changes in the MR, execute the migrations again once the review is complete. -1. Write tests for more complex migrations. -1. If your merge request adds new validations to existing models, to make sure the - data processing is backwards compatible: - - - Ask in the [`#database`](https://gitlab.slack.com/archives/CNZ8E900G) Slack channel - for assistance to execute the database query that checks the existing rows to - ensure existing rows aren't impacted by the change. - - Add the necessary validation with a feature flag to be gradually rolled out - following [the rollout steps](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#rollout). - - If this merge request is urgent, the code owners should make the final call on - whether reviewing existing rows should be included as an immediate follow-up task - to the merge request. - - NOTE: - There isn't a way to know anything about our customers' data on their - [self-managed instances](../../subscriptions/self_managed/index.md), so keep - that in mind for any data implications with your merge request. - -1. Merge requests **must** adhere to the [merge request performance guidelines](../merge_request_performance_guidelines.md). -1. For tests that use Capybara, read - [how to write reliable, asynchronous integration tests](https://thoughtbot.com/blog/write-reliable-asynchronous-integration-tests-with-capybara). -1. If your merge request introduces changes that require additional steps when - installing GitLab from source, add them to `doc/install/installation.md` in - the same merge request. -1. If your merge request introduces changes that require additional steps when - upgrading GitLab from source, add them to - `doc/update/upgrading_from_source.md` in the same merge request. If these - instructions are specific to a version, add them to the "Version specific - upgrading instructions" section. 1. Read and adhere to [The responsibility of the merge request author](../code_review.md#the-responsibility-of-the-merge-request-author). 1. Read and follow [Having your merge request reviewed](../code_review.md#having-your-merge-request-reviewed). +1. Make sure the merge request meets the [Definition of done](#definition-of-done). If you would like quick feedback on your merge request feel free to mention someone from the [core team](https://about.gitlab.com/community/core-team/) or one of the @@ -172,7 +119,7 @@ Commit messages should follow the guidelines below, for reasons explained by Chr #### Why these standards matter 1. Consistent commit messages that follow these guidelines make the history more readable. -1. Concise standard commit messages helps to identify [breaking changes](index.md#breaking-changes) for a deployment or ~"master:broken" quicker when +1. Concise standard commit messages helps to identify [breaking changes](../deprecation_guidelines/index.md) for a deployment or ~"master:broken" quicker when reviewing commits between two points in time. #### Commit message template @@ -218,12 +165,12 @@ the contribution acceptance criteria below: exposing a bug in existing code). Every new class should have corresponding unit tests, even if the class is exercised at a higher level, such as a feature test. - If a failing CI build seems to be unrelated to your contribution, you can try - restarting the failing CI job, rebasing from `main` to bring in updates that + restarting the failing CI job, rebasing on top of target branch to bring in updates that may resolve the failure, or if it has not been fixed yet, ask a developer to help you fix the test. -1. The MR initially contains a few logically organized commits. +1. The MR contains a few logically organized commits, or has [squashing commits enabled](../../user/project/merge_requests/squash_and_merge.md#squash-and-merge). 1. The changes can merge without problems. If not, you should rebase if you're the - only one working on your feature branch, otherwise merge `main`. + only one working on your feature branch, otherwise merge the default branch into the MR branch. 1. Only one specific issue is fixed or one specific feature is implemented. Do not combine things; send separate merge requests for each issue or feature. 1. Migrations should do only one thing (for example, create a table, move data to a new @@ -258,13 +205,60 @@ requirements. ### MR Merge -1. Clear description explaining the relevancy of the contribution. +1. Clear title and description explaining the relevancy of the contribution. 1. Working and clean code that is commented where needed. -1. [Unit, integration, and system tests](../testing_guide/index.md) that all pass - on the CI server. -1. Peer member testing is optional but recommended when the risk of a change is high. This includes when the changes are [far-reaching](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work) or are for [components critical for security](../code_review.md#security). -1. Regressions and bugs are covered with tests that reduce the risk of the issue happening - again. +1. The change is evaluated to [limit the impact of far-reaching work](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work). +1. Testing: + + - [Unit, integration, and system tests](../testing_guide/index.md) that all pass + on the CI server. + - Peer member testing is optional but recommended when the risk of a change is high. + This includes when the changes are [far-reaching](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work) + or are for [components critical for security](../code_review.md#security). + - Description includes any steps or setup required to ensure reviewers can view the changes you've made (for example, include any information about feature flags). + - Regressions and bugs are covered with tests that reduce the risk of the issue happening + again. + - For tests that use Capybara, read + [how to write reliable, asynchronous integration tests](https://thoughtbot.com/blog/write-reliable-asynchronous-integration-tests-with-capybara). + - [Black-box tests/end-to-end tests](../testing_guide/testing_levels.md#black-box-tests-at-the-system-level-aka-end-to-end-tests) + added if required. Please contact [the quality team](https://about.gitlab.com/handbook/engineering/quality/#teams) + with any questions. + - The change is tested in a review app where possible and if appropriate. +1. In case of UI changes: + + - Use available components from the GitLab Design System, + [Pajamas](https://design.gitlab.com/). + - The MR must include *Before* and *After* screenshots if UI changes are made. + - If the MR changes CSS classes, please include the list of affected pages, which + can be found by running `grep css-class ./app -R`. +1. If your MR touches code that executes shell commands, reads or opens files, or + handles paths to files on disk, make sure it adheres to the + [shell command guidelines](../shell_commands.md) +1. [Code changes should include observability instrumentation](../code_review.md#observability-instrumentation). +1. If your code needs to handle file storage, see the [uploads documentation](../uploads/index.md). +1. If your merge request adds one or more migrations: + - Make sure to execute all migrations on a fresh database before the MR is reviewed. + If the review leads to large changes in the MR, execute the migrations again + after the review is complete. + - Write tests for more complex migrations. +1. If your merge request adds new validations to existing models, to make sure the + data processing is backwards compatible: + + - Ask in the [`#database`](https://gitlab.slack.com/archives/CNZ8E900G) Slack channel + for assistance to execute the database query that checks the existing rows to + ensure existing rows aren't impacted by the change. + - Add the necessary validation with a feature flag to be gradually rolled out + following [the rollout steps](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#rollout). + + If this merge request is urgent, the code owners should make the final call on + whether reviewing existing rows should be included as an immediate follow-up task + to the merge request. + + NOTE: + There isn't a way to know anything about our customers' data on their + [self-managed instances](../../subscriptions/self_managed/index.md), so keep + that in mind for any data implications with your merge request. + 1. Code affected by a feature flag is covered by [automated tests with the feature flag enabled and disabled](../feature_flags/index.md#feature-flags-in-tests), or both states are tested as part of peer member testing or as part of the rollout plan. 1. [Performance guidelines](../merge_request_performance_guidelines.md) have been followed. @@ -272,16 +266,22 @@ requirements. 1. [Application and rate limit guidelines](../merge_request_application_and_rate_limit_guidelines.md) have been followed. 1. [Documented](../documentation/index.md) in the `/doc` directory. 1. [Changelog entry added](../changelog.md), if necessary. +1. If your merge request introduces changes that require additional steps when + installing GitLab from source, add them to `doc/install/installation.md` in + the same merge request. +1. If your merge request introduces changes that require additional steps when + upgrading GitLab from source, add them to + `doc/update/upgrading_from_source.md` in the same merge request. If these + instructions are specific to a version, add them to the "Version specific + upgrading instructions" section. 1. Reviewed by relevant reviewers, and all concerns are addressed for Availability, Regressions, and Security. Documentation reviews should take place as soon as possible, but they should not block a merge request. 1. The [MR acceptance checklist](../code_review.md#acceptance-checklist) has been checked as confirmed in the MR. -1. Create an issue in the [infrastructure issue tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues) to inform the Infrastructure department when your contribution is changing default settings or introduces a new setting, if relevant. -1. [Black-box tests/end-to-end tests](../testing_guide/testing_levels.md#black-box-tests-at-the-system-level-aka-end-to-end-tests) - added if required. Please contact [the quality team](https://about.gitlab.com/handbook/engineering/quality/#teams) - with any questions. -1. The change is tested in a review app where possible and if appropriate. -1. The new feature does not degrade the user experience of the product. -1. The change is evaluated to [limit the impact of far-reaching work](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work). +1. Create an issue in the [infrastructure issue tracker](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues) to inform the Infrastructure department when your contribution is changing default settings or introduces a new setting, if relevant. 1. An agreed-upon [rollout plan](https://about.gitlab.com/handbook/engineering/development/processes/rollout-plans/). +1. Your merge request has at least 1 approval, but depending on your changes + you might need additional approvals. Refer to the [Approval guidelines](../code_review.md#approval-guidelines). + - You don't have to select any specific approvers, but you can if you really want + specific people to approve your merge request. 1. Merged by a project maintainer. ### Production use @@ -319,8 +319,8 @@ request: We allow engineering time to fix small problems (with or without an issue) that are incremental improvements, such as: -1. Unprioritized bug fixes (for example, [Banner alerting of project move is -showing up everywhere](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18985)) +1. Unprioritized bug fixes (for example, + [Banner alerting of project move is showing up everywhere](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18985)) 1. Documentation improvements 1. RuboCop or Code Quality improvements diff --git a/doc/development/creating_enums.md b/doc/development/creating_enums.md index 450cb97d978..d3892c4c44e 100644 --- a/doc/development/creating_enums.md +++ b/doc/development/creating_enums.md @@ -1,154 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/creating_enums.md' +remove_date: '2022-11-06' --- -# Creating enums +This document was moved to [another location](database/creating_enums.md). -When creating a new enum, it should use the database type `SMALLINT`. -The `SMALLINT` type size is 2 bytes, which is sufficient for an enum. -This would help to save space in the database. - -To use this type, add `limit: 2` to the migration that creates the column. - -Example: - -```ruby -def change - add_column :ci_job_artifacts, :file_format, :integer, limit: 2 -end -``` - -## All of the key/value pairs should be defined in FOSS - -**Summary:** All enums needs to be defined in FOSS, if a model is also part of the FOSS. - -```ruby -class Model < ApplicationRecord - enum platform: { - aws: 0, - gcp: 1 # EE-only - } -end -``` - -When you add a new key/value pair to a `enum` and if it's EE-specific, you might be -tempted to organize the `enum` as the following: - -```ruby -# Define `failure_reason` enum in `Pipeline` model: -class Pipeline < ApplicationRecord - enum failure_reason: Enums::Pipeline.failure_reasons -end -``` - -```ruby -# Define key/value pairs that used in FOSS and EE: -module Enums - module Pipeline - def self.failure_reasons - { unknown_failure: 0, config_error: 1 } - end - end -end - -Enums::Pipeline.prepend_mod_with('Enums::Pipeline') -``` - -```ruby -# Define key/value pairs that used in EE only: -module EE - module Enums - module Pipeline - override :failure_reasons - def failure_reasons - super.merge(activity_limit_exceeded: 2) - end - end - end -end -``` - -This works as-is, however, it has a couple of downside that: - -- Someone could define a key/value pair in EE that is **conflicted** with a value defined in FOSS. - For example, define `activity_limit_exceeded: 1` in `EE::Enums::Pipeline`. -- When it happens, the feature works totally different. - For example, we cannot figure out `failure_reason` is either `config_error` or `activity_limit_exceeded`. -- When it happens, we have to ship a database migration to fix the data integrity, - which might be impossible if you cannot recover the original value. - -Also, you might observe a workaround for this concern by setting an offset in EE's values. -For example, this example sets `1000` as the offset: - -```ruby -module EE - module Enums - module Pipeline - override :failure_reasons - def failure_reasons - super.merge(activity_limit_exceeded: 1_000, size_limit_exceeded: 1_001) - end - end - end -end -``` - -This looks working as a workaround, however, this approach has some downsides that: - -- Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. - For example, when you move `activity_limit_exceeded` to FOSS, you see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. -- The integer column for the `enum` is likely created [as `SMALLINT`](#creating-enums). - Therefore, you need to be careful of that the offset doesn't exceed the maximum value of 2 bytes integer. - -As a conclusion, you should define all of the key/value pairs in FOSS. -For example, you can simply write the following code in the above case: - -```ruby -class Pipeline < ApplicationRecord - enum failure_reason: { - unknown_failure: 0, - config_error: 1, - activity_limit_exceeded: 2 - } -end -``` - -## Add new values in the gap - -After merging some EE and FOSS enums, there might be a gap between the two groups of values: - -```ruby -module Enums - module Ci - module CommitStatus - def self.failure_reasons - { - # ... - data_integrity_failure: 12, - forward_deployment_failure: 13, - insufficient_bridge_permissions: 1_001, - downstream_bridge_project_not_found: 1_002, - # ... - } - end - end - end -end -``` - -To add new values, you should fill the gap first. -In the example above add `14` instead of `1_003`: - -```ruby -{ - # ... - data_integrity_failure: 12, - forward_deployment_failure: 13, - a_new_value: 14, - insufficient_bridge_permissions: 1_001, - downstream_bridge_project_not_found: 1_002, - # ... -} -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index 7a18da2223f..8a8fe3c0a1e 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Add a foreign key constraint to an existing column -Foreign keys ensure consistency between related database tables. The current database review process **always** encourages you to add [foreign keys](../foreign_keys.md) when creating tables that reference records from other tables. +Foreign keys ensure consistency between related database tables. The current database review process **always** encourages you to add [foreign keys](foreign_keys.md) when creating tables that reference records from other tables. Starting with Rails version 4, Rails includes migration helpers to add foreign key constraints to database tables. Before Rails 4, the only way for ensuring some level of consistency was the @@ -95,7 +95,7 @@ The approach here depends on the data volume and the cleanup strategy. If we can records by doing a database query and the record count is not high, then the data migration can be executed in a Rails migration. -In case the data volume is higher (>1000 records), it's better to create a background migration. If unsure, please contact the database team for advice. +In case the data volume is higher (>1000 records), it's better to create a background migration. If unsure, contact the database team for advice. Example for cleaning up records in the `emails` table in a database migration: diff --git a/doc/development/database/adding_database_indexes.md b/doc/development/database/adding_database_indexes.md new file mode 100644 index 00000000000..8abd7c8298e --- /dev/null +++ b/doc/development/database/adding_database_indexes.md @@ -0,0 +1,410 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Adding Database Indexes + +Indexes can be used to speed up database queries, but when should you add a new +index? Traditionally the answer to this question has been to add an index for +every column used for filtering or joining data. For example, consider the +following query: + +```sql +SELECT * +FROM projects +WHERE user_id = 2; +``` + +Here we are filtering by the `user_id` column and as such a developer may decide +to index this column. + +While in certain cases indexing columns using the above approach may make sense, +it can actually have a negative impact. Whenever you write data to a table, any +existing indexes must also be updated. The more indexes there are, the slower this +can potentially become. Indexes can also take up significant disk space, depending +on the amount of data indexed and the index type. For example, PostgreSQL offers +`GIN` indexes which can be used to index certain data types that cannot be +indexed by regular B-tree indexes. These indexes, however, generally take up more +data and are slower to update compared to B-tree indexes. + +Because of all this, it's important make the following considerations +when adding a new index: + +1. Do the new queries re-use as many existing indexes as possible? +1. Is there enough data that using an index is faster than iterating over + rows in the table? +1. Is the overhead of maintaining the index worth the reduction in query + timings? + +## Re-using Queries + +The first step is to make sure your query re-uses as many existing indexes as +possible. For example, consider the following query: + +```sql +SELECT * +FROM todos +WHERE user_id = 123 +AND state = 'open'; +``` + +Now imagine we already have an index on the `user_id` column but not on the +`state` column. One may think this query performs badly due to `state` being +unindexed. In reality the query may perform just fine given the index on +`user_id` can filter out enough rows. + +The best way to determine if indexes are re-used is to run your query using +`EXPLAIN ANALYZE`. Depending on the joined tables and the columns being used for filtering, +you may find an extra index doesn't make much, if any, difference. + +In short: + +1. Try to write your query in such a way that it re-uses as many existing + indexes as possible. +1. Run the query using `EXPLAIN ANALYZE` and study the output to find the most + ideal query. + +## Data Size + +A database may not use an index even when a regular sequence scan +(iterating over all rows) is faster, especially for small tables. + +Consider adding an index if a table is expected to grow, and your query has to filter a lot of rows. +You may _not_ want to add an index if the table size is small (<`1,000` records), +or if existing indexes already filter out enough rows. + +## Maintenance Overhead + +Indexes have to be updated on every table write. In the case of PostgreSQL, _all_ +existing indexes are updated whenever data is written to a table. As a +result, having many indexes on the same table slows down writes. It's therefore important +to balance query performance with the overhead of maintaining an extra index. + +Let's say that adding an index reduces SELECT timings by 5 milliseconds but increases +INSERT/UPDATE/DELETE timings by 10 milliseconds. In this case, the new index may not be worth +it. A new index is more valuable when SELECT timings are reduced and INSERT/UPDATE/DELETE +timings are unaffected. + +## Finding Unused Indexes + +To see which indexes are unused you can run the following query: + +```sql +SELECT relname as table_name, indexrelname as index_name, idx_scan, idx_tup_read, idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::regclass)) +FROM pg_stat_all_indexes +WHERE schemaname = 'public' +AND idx_scan = 0 +AND idx_tup_read = 0 +AND idx_tup_fetch = 0 +ORDER BY pg_relation_size(indexrelname::regclass) desc; +``` + +This query outputs a list containing all indexes that are never used and sorts +them by indexes sizes in descending order. This query helps in +determining whether existing indexes are still required. More information on +the meaning of the various columns can be found at +<https://www.postgresql.org/docs/current/monitoring-stats.html>. + +To determine if an index is still being used on production, use the following +Thanos query with your index name: + +```sql +sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) +``` + +Because the query output relies on the actual usage of your database, it +may be affected by factors such as: + +- Certain queries never being executed, thus not being able to use certain + indexes. +- Certain tables having little data, resulting in PostgreSQL using sequence + scans instead of index scans. + +This data is only reliable for a frequently used database with +plenty of data, and using as many GitLab features as possible. + +## Requirements for naming indexes + +Indexes with complex definitions must be explicitly named rather than +relying on the implicit naming behavior of migration methods. In short, +that means you **must** provide an explicit name argument for an index +created with one or more of the following options: + +- `where` +- `using` +- `order` +- `length` +- `type` +- `opclass` + +### Considerations for index names + +Check our [Constraints naming conventions](constraint_naming_convention.md) page. + +### Why explicit names are required + +As Rails is database agnostic, it generates an index name only +from the required options of all indexes: table name and column names. +For example, imagine the following two indexes are created in a migration: + +```ruby +def up + add_index :my_table, :my_column + + add_index :my_table, :my_column, where: 'my_column IS NOT NULL' +end +``` + +Creation of the second index would fail, because Rails would generate +the same name for both indexes. + +This naming issue is further complicated by the behavior of the `index_exists?` method. +It considers only the table name, column names, and uniqueness specification +of the index when making a comparison. Consider: + +```ruby +def up + unless index_exists?(:my_table, :my_column, where: 'my_column IS NOT NULL') + add_index :my_table, :my_column, where: 'my_column IS NOT NULL' + end +end +``` + +The call to `index_exists?` returns true if **any** index exists on +`:my_table` and `:my_column`, and index creation is bypassed. + +The `add_concurrent_index` helper is a requirement for creating indexes +on populated tables. Because it cannot be used inside a transactional +migration, it has a built-in check that detects if the index already +exists. In the event a match is found, index creation is skipped. +Without an explicit name argument, Rails can return a false positive +for `index_exists?`, causing a required index to not be created +properly. By always requiring a name for certain types of indexes, the +chance of error is greatly reduced. + +## Temporary indexes + +There may be times when an index is only needed temporarily. + +For example, in a migration, a column of a table might be conditionally +updated. To query which columns must be updated in the +[query performance guidelines](query_performance.md), an index is needed +that would otherwise not be used. + +In these cases, consider a temporary index. To specify a +temporary index: + +1. Prefix the index name with `tmp_` and follow the [naming conventions](constraint_naming_convention.md). +1. Create a follow-up issue to remove the index in the next (or future) milestone. +1. Add a comment in the migration mentioning the removal issue. + +A temporary migration would look like: + +```ruby +INDEX_NAME = 'tmp_index_projects_on_owner_where_emails_disabled' + +def up + # Temporary index to be removed in 13.9 https://gitlab.com/gitlab-org/gitlab/-/issues/1234 + add_concurrent_index :projects, :creator_id, where: 'emails_disabled = false', name: INDEX_NAME +end + +def down + remove_concurrent_index_by_name :projects, INDEX_NAME +end +``` + +## Create indexes asynchronously + +For very large tables, index creation can be a challenge to manage. +While `add_concurrent_index` creates indexes in a way that does not block +normal traffic, it can still be problematic when index creation runs for +many hours. Necessary database operations like `autovacuum` cannot run, and +on GitLab.com, the deployment process is blocked waiting for index +creation to finish. + +To limit impact on GitLab.com, a process exists to create indexes +asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, +index creation can proceed at a lower level of risk. + +### Schedule index creation for a low-impact time + +1. [Schedule the index to be created](#schedule-the-index-to-be-created). +1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). +1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). + +### Schedule the index to be created + +Create an MR with a post-deployment migration which prepares the index +for asynchronous creation. An example of creating an index using +the asynchronous index helpers can be seen in the block below. This migration +enters the index name and definition into the `postgres_async_indexes` +table. The process that runs on weekends pulls indexes from this +table and attempt to create them. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +def up + prepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end + +def down + unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end +``` + +### Verify the MR was deployed and the index exists in production + +You can verify if the post-deploy migration was executed on GitLab.com by: + +- Executing `/chatops run auto_deploy status <merge_sha>`. If the output returns `db/gprd`, + the post-deploy migration has been executed in the production database. More details in this + [guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). +- Use a meta-command in #database-lab, such as: `\d <index_name>`. + - Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). +- Ask someone in #database to check if the index exists. +- With proper access, you can also verify directly on production or in a + production clone. + +### Add a migration to create the index synchronously + +After the index is verified to exist on the production database, create a second +merge request that adds the index synchronously. The schema changes must be +updated and committed to `structure.sql` in this second merge request. +The synchronous migration results in a no-op on GitLab.com, but you should still add the +migration as expected for other installations. The below block +demonstrates how to create the second migration for the previous +asynchronous example. + +**WARNING:** +Verify that the index exists in production before merging a second migration with `add_concurrent_index`. +If the second migration is deployed before the index has been created, +the index is created synchronously when the second migration executes. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +disable_ddl_transaction! + +def up + add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME +end + +def down + remove_concurrent_index_by_name :ci_builds, INDEX_NAME +end +``` + +## Test database index changes locally + +You must test the database index changes locally before creating a merge request. + +### Verify indexes created asynchronously + +Use the asynchronous index helpers on your local environment to test changes for creating an index: + +1. Enable the feature flags by running `Feature.enable(:database_async_index_creation)` and `Feature.enable(:database_reindexing)` in the Rails console. +1. Run `bundle exec rails db:migrate` so that it creates an entry in the `postgres_async_indexes` table. +1. Run `bundle exec rails gitlab:db:reindex` so that the index is created asynchronously. +1. To verify the index, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d <index_name>` to check that your newly created index exists. + +## Drop indexes asynchronously + +For very large tables, index destruction can be a challenge to manage. +While `remove_concurrent_index` removes indexes in a way that does not block +normal traffic, it can still be problematic if index destruction runs for +during autovacuum. Necessary database operations like `autovacuum` cannot run, and +the deployment process on GitLab.com is blocked while waiting for index +destruction to finish. + +To limit the impact on GitLab.com, use the following process to remove indexes +asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, +index destruction can proceed at a lower level of risk. + +1. [Schedule the index to be removed](#schedule-the-index-to-be-removed). +1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production). +1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously). + +### Schedule the index to be removed + +Create an MR with a post-deployment migration which prepares the index +for asynchronous destruction. For example. to destroy an index using +the asynchronous index helpers: + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +def up + prepare_async_index_removal :ci_builds, :some_column, name: INDEX_NAME +end + +def down + unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME +end +``` + +This migration enters the index name and definition into the `postgres_async_indexes` +table. The process that runs on weekends pulls indexes from this table and attempt +to remove them. + +You must test the database index changes locally before creating a merge request. + +### Verify the MR was deployed and the index exists in production + +You can verify if the MR was deployed to GitLab.com with +`/chatops run auto_deploy status <merge_sha>`. To verify the existence of +the index, you can: + +- Use a meta-command in `#database-lab`, for example: `\d <index_name>`. + - Make sure the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). +- Ask someone in `#database` to check if the index exists. +- If you have access, you can verify directly on production or in a + production clone. + +### Add a migration to destroy the index synchronously + +After you verify the index exists in the production database, create a second +merge request that removes the index synchronously. The schema changes must be +updated and committed to `structure.sql` in this second merge request. +The synchronous migration results in a no-op on GitLab.com, but you should still add the +migration as expected for other installations. For example, to +create the second migration for the previous asynchronous example: + +**WARNING:** +Verify that the index no longer exist in production before merging a second migration with `remove_concurrent_index_by_name`. +If the second migration is deployed before the index has been destroyed, +the index is destroyed synchronously when the second migration executes. + +```ruby +# in db/post_migrate/ + +INDEX_NAME = 'index_ci_builds_on_some_column' + +disable_ddl_transaction! + +def up + remove_concurrent_index_by_name :ci_builds, name: INDEX_NAME +end + +def down + add_concurrent_index :ci_builds, :some_column, INDEX_NAME +end +``` + +### Verify indexes removed asynchronously + +To test changes for removing an index, use the asynchronous index helpers on your local environment: + +1. Enable the feature flags by running `Feature.enable(:database_async_index_destruction)` and `Feature.enable(:database_reindexing)` in the Rails console. +1. Run `bundle exec rails db:migrate` which should create an entry in the `postgres_async_indexes` table. +1. Run `bundle exec rails gitlab:db:reindex` destroy the index asynchronously. +1. To verify the index, open the PostgreSQL console by using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) + command `gdk psql` and run `\d <index_name>` to check that the destroyed index no longer exists. diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index 2d079656e23..79c76b351c8 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -93,9 +93,8 @@ class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.0] end ``` -You can consider [enabling lock retries]( -https://docs.gitlab.com/ee/development/migration_style_guide.html#usage-with-transactional-migrations -) when you run a migration on big tables, because it might take some time to +You can consider [enabling lock retries](../migration_style_guide.md#usage-with-transactional-migrations) +when you run a migration on big tables, because it might take some time to acquire a lock on this table. #### B. The removed column has an index or constraint that belongs to it @@ -104,7 +103,7 @@ If the `down` method requires adding back any dropped indexes or constraints, th be done within a transactional migration, then the migration would look like this: ```ruby -class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[1.0] +class RemoveUsersUpdatedAtColumn < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up @@ -126,13 +125,11 @@ end In the `down` method, we check to see if the column already exists before adding it again. We do this because the migration is non-transactional and might have failed while it was running. -The [`disable_ddl_transaction!`]( -https://docs.gitlab.com/ee/development/migration_style_guide.html#usage-with-non-transactional-migrations-disable_ddl_transaction -) is used to disable the transaction that wraps the whole migration. +The [`disable_ddl_transaction!`](../migration_style_guide.md#usage-with-non-transactional-migrations-disable_ddl_transaction) +is used to disable the transaction that wraps the whole migration. -You can refer to the page [Migration Style Guide]( -https://docs.gitlab.com/ee/development/migration_style_guide.html -) for more information about database migrations. +You can refer to the page [Migration Style Guide](../migration_style_guide.md) +for more information about database migrations. ### Step 3: Removing the ignore rule (release M+2) @@ -145,9 +142,13 @@ the `remove_after` date has passed. ## Renaming Columns Renaming columns the normal way requires downtime as an application may continue -using the old column name during/after a database migration. To rename a column -without requiring downtime we need two migrations: a regular migration, and a -post-deployment migration. Both these migration can go in the same release. +to use the old column names during or after a database migration. To rename a column +without requiring downtime, we need two migrations: a regular migration and a +post-deployment migration. Both these migrations can go in the same release. + +NOTE: +It's not possible to rename columns with default values. For more details, see +[this merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52032#default-values). ### Step 1: Add The Regular Migration @@ -157,7 +158,7 @@ renaming. For example ```ruby # A regular migration in db/migrate -class RenameUsersUpdatedAtToUpdatedAtTimestamp < Gitlab::Database::Migration[1.0] +class RenameUsersUpdatedAtToUpdatedAtTimestamp < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up @@ -185,7 +186,7 @@ We can perform this cleanup using ```ruby # A post-deployment migration in db/post_migrate -class CleanupUsersUpdatedAtRename < Gitlab::Database::Migration[1.0] +class CleanupUsersUpdatedAtRename < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up @@ -198,7 +199,7 @@ class CleanupUsersUpdatedAtRename < Gitlab::Database::Migration[1.0] end ``` -If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. +If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet. With [Canary](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/) it is possible that the system runs in this state for a significant amount of time. ## Changing Column Constraints @@ -232,7 +233,7 @@ as follows: ```ruby # A regular migration in db/migrate -class ChangeUsersUsernameStringToText < Gitlab::Database::Migration[1.0] +class ChangeUsersUsernameStringToText < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up @@ -251,7 +252,7 @@ Next we need to clean up our changes using a post-deployment migration: ```ruby # A post-deployment migration in db/post_migrate -class ChangeUsersUsernameStringToTextCleanup < Gitlab::Database::Migration[1.0] +class ChangeUsersUsernameStringToTextCleanup < Gitlab::Database::Migration[2.0] disable_ddl_transaction! def up @@ -295,8 +296,7 @@ when migrating a column in a large table (for example, `issues`). Background migrations spread the work / load over a longer time period, without slowing down deployments. -For more information, see [the documentation on cleaning up background -migrations](background_migrations.md#cleaning-up). +For more information, see [the documentation on cleaning up background migrations](background_migrations.md#cleaning-up). ## Adding Indexes diff --git a/doc/development/database/background_migrations.md b/doc/development/database/background_migrations.md index 0124dbae51f..9b596eb7379 100644 --- a/doc/development/database/background_migrations.md +++ b/doc/development/database/background_migrations.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: Background migrations are strongly discouraged in favor of the new [batched background migrations framework](batched_background_migrations.md). -Please check that documentation and determine if that framework suits your needs and fall back +Check that documentation and determine if that framework suits your needs and fall back to these only if required. Background migrations should be used to perform data migrations whenever a @@ -368,9 +368,9 @@ A strategy to make the migration run faster is to schedule larger batches, and t within the background migration to perform multiple statements. The background migration helpers that queue multiple jobs such as -`queue_background_migration_jobs_by_range_at_intervals` use [`EachBatch`](../iterating_tables_in_batches.md). +`queue_background_migration_jobs_by_range_at_intervals` use [`EachBatch`](iterating_tables_in_batches.md). The example above has batches of 1000, where each queued job takes two seconds. If the query has been optimized -to make the time for the delete statement within the [query performance guidelines](../query_performance.md), +to make the time for the delete statement within the [query performance guidelines](query_performance.md), 1000 may be the largest number of records that can be deleted in a reasonable amount of time. The minimum and most common interval for delaying jobs is two minutes. This results in two seconds diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index f3ea82b5c61..edb22fcf436 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -105,11 +105,16 @@ for more details. ## Batched background migrations for EE-only features -All the background migration classes for EE-only features should be present in GitLab CE. -For this purpose, create an empty class for GitLab CE, and extend it for GitLab EE +All the background migration classes for EE-only features should be present in GitLab FOSS. +For this purpose, create an empty class for GitLab FOSS, and extend it for GitLab EE as explained in the guidelines for [implementing Enterprise Edition features](../ee_features.md#code-in-libgitlabbackground_migration). +NOTE: +Background migration classes for EE-only features that use job arguments should define them +in the GitLab FOSS class. This is required to prevent job arguments validation from failing when +migration is scheduled in GitLab FOSS context. + Batched Background migrations are simple classes that define a `perform` method. A Sidekiq worker then executes such a class, passing any arguments to it. All migration classes must be defined in the namespace @@ -132,6 +137,10 @@ queue_batched_background_migration( ) ``` +NOTE: +This helper raises an error if the number of provided job arguments does not match +the number of [job arguments](#job-arguments) defined in `JOB_CLASS_NAME`. + Make sure the newly-created data is either migrated, or saved in both the old and new version upon creation. Removals in turn can be handled by defining foreign keys with cascading deletes. @@ -186,6 +195,115 @@ Bump to the [import/export version](../../user/project/settings/import_export.md be required, if importing a project from a prior version of GitLab requires the data to be in the new format. +## Job arguments + +`BatchedMigrationJob` provides the `job_arguments` helper method for job classes to define the job arguments they need. + +Batched migrations scheduled with `queue_batched_background_migration` **must** use the helper to define the job arguments: + +```ruby +queue_batched_background_migration( + 'CopyColumnUsingBackgroundMigrationJob', + TABLE_NAME, + 'name', 'name_convert_to_text', + job_interval: DELAY_INTERVAL +) +``` + +NOTE: +If the number of defined job arguments does not match the number of job arguments provided when +scheduling the migration, `queue_batched_background_migration` raises an error. + +In this example, `copy_from` returns `name`, and `copy_to` returns `name_convert_to_text`: + +```ruby +class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob + job_arguments :copy_from, :copy_to + + def perform + from_column = connection.quote_column_name(copy_from) + to_column = connection.quote_column_name(copy_to) + + assignment_clause = "#{to_column} = #{from_column}" + + each_sub_batch(operation_name: :update_all) do |relation| + relation.update_all(assignment_clause) + end + end +end +``` + +### Additional filters + +By default, when creating background jobs to perform the migration, batched background migrations +iterate over the full specified table. This iteration is done using the +[`PrimaryKeyBatchingStrategy`](https://gitlab.com/gitlab-org/gitlab/-/blob/c9dabd1f4b8058eece6d8cb4af95e9560da9a2ee/lib/gitlab/database/migrations/batched_background_migration_helpers.rb#L17). If the table has 1000 records +and the batch size is 100, the work is batched into 10 jobs. For illustrative purposes, +`EachBatch` is used like this: + +```ruby +# PrimaryKeyBatchingStrategy +Namespace.each_batch(of: 100) do |relation| + relation.where(type: nil).update_all(type: 'User') # this happens in each background job +end +``` + +In some cases, only a subset of records must be examined. If only 10% of the 1000 records +need examination, apply a filter to the initial relation when the jobs are created: + +```ruby +Namespace.where(type: nil).each_batch(of: 100) do |relation| + relation.update_all(type: 'User') +end +``` + +In the first example, we don't know how many records will be updated in each batch. +In the second (filtered) example, we know exactly 100 will be updated with each batch. + +`BatchedMigrationJob` provides a `scope_to` helper method to apply additional filters and achieve this: + +1. Create a new migration job class that inherits from `BatchedMigrationJob` and defines the additional filter: + + ```ruby + class BackfillNamespaceType < BatchedMigrationJob + scope_to ->(relation) { relation.where(type: nil) } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + sub_batch.update_all(type: 'User') + end + end + end + ``` + +1. In the post-deployment migration, enqueue the batched background migration: + + ```ruby + class BackfillNamespaceType < Gitlab::Database::Migration[2.0] + MIGRATION = 'BackfillNamespaceType' + DELAY_INTERVAL = 2.minutes + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :namespaces, + :id, + job_interval: DELAY_INTERVAL + ) + end + + def down + delete_batched_background_migration(MIGRATION, :namespaces, :id, []) + end + end + ``` + +NOTE: +When applying additional filters, it is important to ensure they are properly covered by an index to optimize `EachBatch` performance. +In the example above we need an index on `(type, id)` to support the filters. See [the `EachBatch` docs for more information](../iterating_tables_in_batches.md). + ## Example The `routes` table has a `source_type` field that's used for a polymorphic relationship. @@ -221,8 +339,6 @@ background migration. correctly handled by the batched migration framework. Any subclass of `BatchedMigrationJob` is initialized with necessary arguments to execute the batch, as well as a connection to the tracking database. - Additional `job_arguments` set on the migration are passed to the - job's `perform` method. 1. Add a new trigger to the database to update newly created and updated routes, similar to this example: @@ -320,7 +436,7 @@ The default batching strategy provides an efficient way to iterate over primary However, if you need to iterate over columns where values are not unique, you must use a different batching strategy. -The `LooseIndexScanBatchingStrategy` batching strategy uses a special version of [`EachBatch`](../iterating_tables_in_batches.md#loose-index-scan-with-distinct_each_batch) +The `LooseIndexScanBatchingStrategy` batching strategy uses a special version of [`EachBatch`](iterating_tables_in_batches.md#loose-index-scan-with-distinct_each_batch) to provide efficient and stable iteration over the distinct column values. This example shows a batched background migration where the `issues.project_id` column is used as @@ -374,76 +490,8 @@ module Gitlab end ``` -### Adding filters to the initial batching - -By default, when creating background jobs to perform the migration, batched background migrations will iterate over the full specified table. This is done using the [`PrimaryKeyBatchingStrategy`](https://gitlab.com/gitlab-org/gitlab/-/blob/c9dabd1f4b8058eece6d8cb4af95e9560da9a2ee/lib/gitlab/database/migrations/batched_background_migration_helpers.rb#L17). This means if there are 1000 records in the table and the batch size is 100, there will be 10 jobs. For illustrative purposes, `EachBatch` is used like this: - -```ruby -# PrimaryKeyBatchingStrategy -Projects.all.each_batch(of: 100) do |relation| - relation.where(foo: nil).update_all(foo: 'bar') # this happens in each background job -end -``` - -There are cases where we only need to look at a subset of records. Perhaps we only need to update 1 out of every 10 of those 1000 records. It would be best if we could apply a filter to the initial relation when the jobs are created: - -```ruby -Projects.where(foo: nil).each_batch(of: 100) do |relation| - relation.update_all(foo: 'bar') -end -``` - -In the `PrimaryKeyBatchingStrategy` example, we do not know how many records will be updated in each batch. In the filtered example, we know exactly 100 will be updated with each batch. - -The `PrimaryKeyBatchingStrategy` contains [a method that can be overwritten](https://gitlab.com/gitlab-org/gitlab/-/blob/dd1e70d3676891025534dc4a1e89ca9383178fe7/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb#L38-52) to apply additional filtering on the initial `EachBatch`. - -We can accomplish this by: - -1. Create a new class that inherits from `PrimaryKeyBatchingStrategy` and overrides the method using the desired filter (this may be the same filter used in the sub-batch): - - ```ruby - # frozen_string_literal: true - - module GitLab - module BackgroundMigration - module BatchingStrategies - class FooStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(foo: nil) - end - end - end - end - end - ``` - -1. In the post-deployment migration that queues the batched background migration, specify the new batching strategy using the `batch_class_name` parameter: - - ```ruby - class BackfillProjectsFoo < Gitlab::Database::Migration[2.0] - MIGRATION = 'BackfillProjectsFoo' - DELAY_INTERVAL = 2.minutes - BATCH_CLASS_NAME = 'FooStrategy' - - restrict_gitlab_migration gitlab_schema: :gitlab_main - - def up - queue_batched_background_migration( - MIGRATION, - :routes, - :id, - job_interval: DELAY_INTERVAL, - batch_class_name: BATCH_CLASS_NAME - ) - end - - def down - delete_batched_background_migration(MIGRATION, :routes, :id, []) - end - end - ``` - -When applying a batching strategy, it is important to ensure the filter properly covered by an index to optimize `EachBatch` performance. See [the `EachBatch` docs for more information](../iterating_tables_in_batches.md). +NOTE: +[Additional filters](#additional-filters) defined with `scope_to` will be ignored by `LooseIndexScanBatchingStrategy` and `distinct_each_batch`. ## Testing diff --git a/doc/development/database/ci_mirrored_tables.md b/doc/development/database/ci_mirrored_tables.md new file mode 100644 index 00000000000..06f0087fafe --- /dev/null +++ b/doc/development/database/ci_mirrored_tables.md @@ -0,0 +1,156 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# CI mirrored tables + +## Problem statement + +As part of the database [decomposition work](https://gitlab.com/groups/gitlab-org/-/epics/6168), +which had the goal of splitting the single database GitLab is using, into two databases: `main` and +`ci`, came the big challenge of +[removing all joins between the `main` and the `ci` tables](multiple_databases.md#removing-joins-between-ci-and-non-ci-tables). +That is because PostgreSQL doesn't support joins between tables that belong to different databases. +However, some core application models in the main database are queried very often by the CI side. +For example: + +- `Namespace`, in the `namespaces` table. +- `Project`, in the `projects` table. + +Not being able to do `joins` on these tables brings a great challenge. The team chose to perform logical +replication of those tables from the main database to the CI database, in the new tables: + +- `ci_namespace_mirrors`, as a mirror of the `namespaces` table +- `ci_project_mirrors`, as a mirror of the `projects` table + +This logical replication means two things: + +1. The `main` database tables can be queried and joined to the `namespaces` and `projects` tables. +1. The `ci` database tables can be joined with the `ci_namespace_mirrors` and `ci_project_mirrors` tables. + +```mermaid +graph LR + + subgraph "Main database (tables)" + A[namespaces] -->|updates| B[namespaces_sync_events] + A -->|deletes| C[loose_foreign_keys_deleted_records] + D[projects] -->|deletes| C + D -->|updates| E[projects_sync_events] + end + + B --> F + C --> G + E --> H + + subgraph "Sidekiq worker jobs" + F[Namespaces::ProcessSyncEventsWorker] + G[LooseForeignKeys::CleanupWorker] + H[Projects::ProcessSyncEventsWorker] + end + + F -->|do update| I + G -->|delete records| I + G -->|delete records| J + H -->|do update| J + + subgraph "CI database (tables)" + I[ci_namespace_mirrors] + J[ci_project_mirrors] + end +``` + +This replication was restricted only to a few attributes that are needed from each model: + +- From `Namespace` we replicate `traversal_ids`. +- From `Project` we replicate only the `namespace_id`, which represents the group which the project belongs to. + +## Keeping the CI mirrored tables in sync with the source tables + +We must care about two type 3 events to keep +the source and the target tables in sync: + +1. Creation of new namespaces or projects. +1. Updating the namespaces or projects. +1. Deleting namespaces/projects. + +```mermaid +graph TD + + subgraph "CI database (tables)" + E[other CI tables] + F{queries with joins allowed} + G[ci_project_mirrors] + H[ci_namespace_mirrors] + + E---F + F---G + F---H + end + + A---B + B---C + B---D + +L["⛔ ← Joins are not allowed → ⛔"] + + subgraph "Main database (tables)" + A[other main tables] + B{queries with joins allowed} + C[projects] + D[namespaces] + end +``` + +### Create and update + +Syncing the data of newly created or updated namespaces or projects happens in this +order: + +1. **On the `main` database**: Any `INSERT` or `UPDATE` on the `namespaces` or `projects` tables + adds an entry to the tables `namespaces_sync_events`, and `projects_sync_events`. These tables + also exist on the `main` database. These entries are added by triggers on both of the tables. +1. **On the model level**: After a commit happens on either of the source models `Namespace` or + `Project`, it schedules the corresponding Sidekiq jobs `Namespaces::ProcessSyncEventsWorker` + or `Projects::ProcessSyncEventsWorker` to run. +1. These workers then: + 1. Read the entries from the tables `(namespaces/project)_sync_events` + from the `main` database, to check which namespaces or projects to sync. + 1. Copy the data for any updated records into the target + tables `ci_namespace_mirrors`, `ci_project_mirrors`. + +### Delete + +When any of `namespaces` or `projects` are deleted, the target records on the mirrored +CI tables are deleted using the [loose foreign keys](loose_foreign_keys.md) (LFK) mechanism. + +By having these items in the `config/gitlab_loose_foreign_keys.yml`, the LFK mechanism +was already working as expected. It deleted any records on the CI mirrored +tables that mapped to deleted `namespaces` or `projects` in the `main` database. + +```yaml +ci_namespace_mirrors: + - table: namespaces + column: namespace_id + on_delete: async_delete +ci_project_mirrors: + - table: projects + column: project_id + on_delete: async_delete +``` + +## Consistency Checking + +To make sure that both syncing mechanisms work as expected, we deploy +two extra worker jobs, triggered by cron jobs every few minutes: + +1. `Database::CiNamespaceMirrorsConsistencyCheckWorker` +1. `Database::CiProjectMirrorsConsistencyCheckWorker` + +These jobs: + +1. Scan both of the source tables on the `main` database, using a cursor. +1. Compare the items in the `namespaces` and `projects` with the target tables on the `ci` database. +1. Report the items that are not in sync to Kibana and Prometheus. +1. Corrects any discrepancies. diff --git a/doc/development/database/client_side_connection_pool.md b/doc/development/database/client_side_connection_pool.md index dc52a551407..3cd0e836a8d 100644 --- a/doc/development/database/client_side_connection_pool.md +++ b/doc/development/database/client_side_connection_pool.md @@ -10,8 +10,8 @@ Ruby processes accessing the database through ActiveRecord, automatically calculate the connection-pool size for the process based on the concurrency. -Because of the way [Ruby on Rails manages database -connections](#connection-lifecycle), it is important that we have at +Because of the way [Ruby on Rails manages database connections](#connection-lifecycle), +it is important that we have at least as many connections as we have threads. While there is a 'pool' setting in [`database.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database.yml.postgresql), it is not very practical because you need to maintain it in tandem with the number of application threads. For this @@ -28,9 +28,8 @@ because connections are instantiated lazily. ## Troubleshooting connection-pool issues -The connection-pool usage can be seen per environment in the [connection-pool -saturation -dashboard](https://dashboards.gitlab.net/d/alerts-sat_rails_db_connection_pool/alerts-rails_db_connection_pool-saturation-detail?orgId=1). +The connection-pool usage can be seen per environment in the +[connection-pool saturation dashboard](https://dashboards.gitlab.net/d/alerts-sat_rails_db_connection_pool/alerts-rails_db_connection_pool-saturation-detail?orgId=1). If the connection-pool is too small, this would manifest in `ActiveRecord::ConnectionTimeoutError`s from the application. Because we alert @@ -41,8 +40,8 @@ hardcoded value (10). At this point, we need to investigate what is using more connections than we anticipated. To do that, we can use the -`gitlab_ruby_threads_running_threads` metric. For example, [this -graph](https://thanos.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum%20by%20(thread_name)%20(%20gitlab_ruby_threads_running_threads%7Buses_db_connection%3D%22yes%22%7D%20)&g0.tab=0) +`gitlab_ruby_threads_running_threads` metric. For example, +[this graph](https://thanos.gitlab.net/graph?g0.range_input=1h&g0.max_source_resolution=0s&g0.expr=sum%20by%20(thread_name)%20(%20gitlab_ruby_threads_running_threads%7Buses_db_connection%3D%22yes%22%7D%20)&g0.tab=0) shows all running threads that connect to the database by their name. Threads labeled `puma worker` or `sidekiq_worker_thread` are the threads that define `Gitlab::Runtime.max_threads` so those are diff --git a/doc/development/database/creating_enums.md b/doc/development/database/creating_enums.md new file mode 100644 index 00000000000..450cb97d978 --- /dev/null +++ b/doc/development/database/creating_enums.md @@ -0,0 +1,154 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Creating enums + +When creating a new enum, it should use the database type `SMALLINT`. +The `SMALLINT` type size is 2 bytes, which is sufficient for an enum. +This would help to save space in the database. + +To use this type, add `limit: 2` to the migration that creates the column. + +Example: + +```ruby +def change + add_column :ci_job_artifacts, :file_format, :integer, limit: 2 +end +``` + +## All of the key/value pairs should be defined in FOSS + +**Summary:** All enums needs to be defined in FOSS, if a model is also part of the FOSS. + +```ruby +class Model < ApplicationRecord + enum platform: { + aws: 0, + gcp: 1 # EE-only + } +end +``` + +When you add a new key/value pair to a `enum` and if it's EE-specific, you might be +tempted to organize the `enum` as the following: + +```ruby +# Define `failure_reason` enum in `Pipeline` model: +class Pipeline < ApplicationRecord + enum failure_reason: Enums::Pipeline.failure_reasons +end +``` + +```ruby +# Define key/value pairs that used in FOSS and EE: +module Enums + module Pipeline + def self.failure_reasons + { unknown_failure: 0, config_error: 1 } + end + end +end + +Enums::Pipeline.prepend_mod_with('Enums::Pipeline') +``` + +```ruby +# Define key/value pairs that used in EE only: +module EE + module Enums + module Pipeline + override :failure_reasons + def failure_reasons + super.merge(activity_limit_exceeded: 2) + end + end + end +end +``` + +This works as-is, however, it has a couple of downside that: + +- Someone could define a key/value pair in EE that is **conflicted** with a value defined in FOSS. + For example, define `activity_limit_exceeded: 1` in `EE::Enums::Pipeline`. +- When it happens, the feature works totally different. + For example, we cannot figure out `failure_reason` is either `config_error` or `activity_limit_exceeded`. +- When it happens, we have to ship a database migration to fix the data integrity, + which might be impossible if you cannot recover the original value. + +Also, you might observe a workaround for this concern by setting an offset in EE's values. +For example, this example sets `1000` as the offset: + +```ruby +module EE + module Enums + module Pipeline + override :failure_reasons + def failure_reasons + super.merge(activity_limit_exceeded: 1_000, size_limit_exceeded: 1_001) + end + end + end +end +``` + +This looks working as a workaround, however, this approach has some downsides that: + +- Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. + For example, when you move `activity_limit_exceeded` to FOSS, you see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. +- The integer column for the `enum` is likely created [as `SMALLINT`](#creating-enums). + Therefore, you need to be careful of that the offset doesn't exceed the maximum value of 2 bytes integer. + +As a conclusion, you should define all of the key/value pairs in FOSS. +For example, you can simply write the following code in the above case: + +```ruby +class Pipeline < ApplicationRecord + enum failure_reason: { + unknown_failure: 0, + config_error: 1, + activity_limit_exceeded: 2 + } +end +``` + +## Add new values in the gap + +After merging some EE and FOSS enums, there might be a gap between the two groups of values: + +```ruby +module Enums + module Ci + module CommitStatus + def self.failure_reasons + { + # ... + data_integrity_failure: 12, + forward_deployment_failure: 13, + insufficient_bridge_permissions: 1_001, + downstream_bridge_project_not_found: 1_002, + # ... + } + end + end + end +end +``` + +To add new values, you should fill the gap first. +In the example above add `14` instead of `1_003`: + +```ruby +{ + # ... + data_integrity_failure: 12, + forward_deployment_failure: 13, + a_new_value: 14, + insufficient_bridge_permissions: 1_001, + downstream_bridge_project_not_found: 1_002, + # ... +} +``` diff --git a/doc/development/database/database_debugging.md b/doc/development/database/database_debugging.md new file mode 100644 index 00000000000..5921dc942f2 --- /dev/null +++ b/doc/development/database/database_debugging.md @@ -0,0 +1,177 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Troubleshooting and Debugging Database + +This section is to help give some copy-pasta you can use as a reference when you +run into some head-banging database problems. + +A first step is to search for your error in Slack, or search for `GitLab <my error>` with Google. + +Available `RAILS_ENV`: + +- `production` (generally not for your main GDK database, but you may need this for other installations such as Omnibus). +- `development` (this is your main GDK db). +- `test` (used for tests like RSpec). + +## Delete everything and start over + +If you just want to delete everything and start over with an empty DB (approximately 1 minute): + +```shell +bundle exec rake db:reset RAILS_ENV=development +``` + +If you want to seed the empty DB with sample data (approximately 4 minutes): + +```shell +bundle exec rake dev:setup +``` + +If you just want to delete everything and start over with sample data (approximately 4 minutes). This +also does `db:reset` and runs DB-specific migrations: + +```shell +bundle exec rake db:setup RAILS_ENV=development +``` + +If your test DB is giving you problems, it is safe to delete everything because it doesn't contain important +data: + +```shell +bundle exec rake db:reset RAILS_ENV=test +``` + +## Migration wrangling + +- `bundle exec rake db:migrate RAILS_ENV=development`: Execute any pending migrations that you may have picked up from a MR +- `bundle exec rake db:migrate:status RAILS_ENV=development`: Check if all migrations are `up` or `down` +- `bundle exec rake db:migrate:down VERSION=20170926203418 RAILS_ENV=development`: Tear down a migration +- `bundle exec rake db:migrate:up VERSION=20170926203418 RAILS_ENV=development`: Set up a migration +- `bundle exec rake db:migrate:redo VERSION=20170926203418 RAILS_ENV=development`: Re-run a specific migration + +## Manually access the database + +Access the database via one of these commands (they all get you to the same place) + +```shell +gdk psql -d gitlabhq_development +bundle exec rails dbconsole -e development +bundle exec rails db -e development +``` + +- `\q`: Quit/exit +- `\dt`: List all tables +- `\d+ issues`: List columns for `issues` table +- `CREATE TABLE board_labels();`: Create a table called `board_labels` +- `SELECT * FROM schema_migrations WHERE version = '20170926203418';`: Check if a migration was run +- `DELETE FROM schema_migrations WHERE version = '20170926203418';`: Manually remove a migration + +## Access the database with a GUI + +Most GUIs (DataGrip, RubyMine, DBeaver) require a TCP connection to the database, but by default +the database runs on a UNIX socket. To be able to access the database from these tools, some steps +are needed: + +1. On the GDK root directory, run: + + ```shell + gdk config set postgresql.host localhost + ``` + +1. Open your `gdk.yml`, and confirm that it has the following lines: + + ```yaml + postgresql: + host: localhost + ``` + +1. Reconfigure GDK: + + ```shell + gdk reconfigure + ``` + +1. On your database GUI, select `localhost` as host, `5432` as port and `gitlabhq_development` as database. + Alternatively, you can use the connection string `postgresql://localhost:5432/gitlabhq_development`. + +The new connection should be working now. + +## Access the GDK database with Visual Studio Code + +Use these instructions for exploring the GitLab database while developing with the GDK: + +1. Install or open [Visual Studio Code](https://code.visualstudio.com/download). +1. Install the [PostgreSQL VSCode Extension](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres). +1. In Visual Studio Code select **PostgreSQL Explorer** in the left toolbar. +1. In the top bar of the new window, select `+` to **Add Database Connection**, and follow the prompts to fill in the details: + 1. **Hostname**: the path to the PostgreSQL folder in your GDK directory (for example `/dev/gitlab-development-kit/postgresql`). + 1. **PostgreSQL user to authenticate as**: usually your local username, unless otherwise specified during PostgreSQL installation. + 1. **Password of the PostgreSQL user**: the password you set when installing PostgreSQL. + 1. **Port number to connect to**: `5432` (default). + 1. **Use an SSL connection?** This depends on your installation. Options are: + - **Use Secure Connection** + - **Standard Connection** (default) + 1. **Optional. The database to connect to**: `gitlabhq_development`. + 1. **The display name for the database connection**: `gitlabhq_development`. + +Your database connection should now be displayed in the PostgreSQL Explorer pane and +you can explore the `gitlabhq_development` database. If you cannot connect, ensure +that GDK is running. For further instructions on how to use the PostgreSQL Explorer +Extension for Visual Studio Code, read the [usage section](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres#usage) +of the extension documentation. + +## FAQ + +### `ActiveRecord::PendingMigrationError` with Spring + +When running specs with the [Spring pre-loader](../rake_tasks.md#speed-up-tests-rake-tasks-and-migrations), +the test database can get into a corrupted state. Trying to run the migration or +dropping/resetting the test database has no effect. + +```shell +$ bundle exec spring rspec some_spec.rb +... +Failure/Error: ActiveRecord::Migration.maintain_test_schema! + +ActiveRecord::PendingMigrationError: + + + Migrations are pending. To resolve this issue, run: + + bin/rake db:migrate RAILS_ENV=test +# ~/.rvm/gems/ruby-2.3.3/gems/activerecord-4.2.10/lib/active_record/migration.rb:392:in `check_pending!' +... +0 examples, 0 failures, 1 error occurred outside of examples +``` + +To resolve, you can kill the spring server and app that lives between spec runs. + +```shell +$ ps aux | grep spring +eric 87304 1.3 2.9 3080836 482596 ?? Ss 10:12AM 4:08.36 spring app | gitlab | started 6 hours ago | test mode +eric 37709 0.0 0.0 2518640 7524 s006 S Wed11AM 0:00.79 spring server | gitlab | started 29 hours ago +$ kill 87304 +$ kill 37709 +``` + +### db:migrate `database version is too old to be migrated` error + +Users receive this error when `db:migrate` detects that the current schema version +is older than the `MIN_SCHEMA_VERSION` defined in the `Gitlab::Database` library +module. + +Over time we cleanup/combine old migrations in the codebase, so it is not always +possible to migrate GitLab from every previous version. + +In some cases you may want to bypass this check. For example, if you were on a version +of GitLab schema later than the `MIN_SCHEMA_VERSION`, and then rolled back the +to an older migration, from before. In this case, to migrate forward again, +you should set the `SKIP_SCHEMA_VERSION_CHECK` environment variable. + +```shell +bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true +``` diff --git a/doc/development/database/database_dictionary.md b/doc/development/database/database_dictionary.md new file mode 100644 index 00000000000..c330c5e67bd --- /dev/null +++ b/doc/development/database/database_dictionary.md @@ -0,0 +1,51 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Database Dictionary + +This page documents the database schema for GitLab, so data analysts and other groups can +locate the feature categories responsible for specific database tables. + +## Location + +Database dictionary metadata files are stored in the `gitlab` project under `db/docs/`. + +## Example dictionary file + +```yaml +--- +table_name: terraform_states +classes: +- Terraform::State +feature_categories: +- infrastructure_as_code +description: Represents a Terraform state backend +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26619 +milestone: '13.0' +``` + +## Schema + +| Attribute | Type | Required | Description | +|----------------------|---------------|----------|--------------------------------------------------------------------------| +| `table_name` | String | yes | Database table name | +| `classes` | Array(String) | no | List of classes that respond to `.table_name` with the `table_name` | +| `feature_categories` | Array(String) | yes | List of feature categories using this table | +| `description` | String | no | Text description of the information stored in the table and it's purpose | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table | +| `milestone` | String | no | The milestone that introduced this table | + +## Adding tables + +When adding a new table, create a new file under `db/docs/` named +`<table_name>.yml` containing as much information as you know about the table. + +Include this file in the commit with the migration that creates the table. + +## Dropping tables + +When dropping a table, you must remove the metadata file from `db/docs/` +in the same commit with the migration that drops the table. diff --git a/doc/development/database/database_lab.md b/doc/development/database/database_lab.md index 5346df2690d..1d584a4ec6f 100644 --- a/doc/development/database/database_lab.md +++ b/doc/development/database/database_lab.md @@ -95,7 +95,7 @@ To connect to a clone using `psql`: 1. In the **Clone details** page of the Postgres.ai web interface, copy and run the command to start SSH port forwarding for the clone. 1. In the **Clone details** page of the Postgres.ai web interface, copy and run the `psql` connection string. - Use the password provided at setup. + Use the password provided at setup and set the `dbname` to `gitlabhq_dblab` (or check what databases are available by using `psql -l` with the same query string but `dbname=postgres`). After you connect, use clone like you would any `psql` console in production, but with the added benefit and safety of an isolated writeable environment. diff --git a/doc/development/database/database_query_comments.md b/doc/development/database/database_query_comments.md new file mode 100644 index 00000000000..2798071bc06 --- /dev/null +++ b/doc/development/database/database_query_comments.md @@ -0,0 +1,62 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Database query comments with Marginalia + +The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add +query comments containing application related context information to PostgreSQL +queries generated by ActiveRecord. + +It is very useful for tracing problematic queries back to the application source. + +An engineer during an on-call incident has the full context of a query +and its application source from the comments. + +## Metadata information in comments + +Queries generated from **Rails** include the following metadata in comments: + +- `application` +- `correlation_id` +- `endpoint_id` +- `line` + +Queries generated from **Sidekiq** workers include the following metadata +in comments: + +- `application` +- `jid` +- `correlation_id` +- `endpoint_id` +- `line` + +`endpoint_id` is a single field that can represent any endpoint in the application: + +- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`. +- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`. +- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`. + +`line` is not present in production logs due to the additional overhead required. + +Examples of queries with comments: + +- Rails: + + ```sql + /*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1 + ``` + +- Grape: + + ```sql + /*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1 + ``` + +- Sidekiq: + + ```sql + /*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3 + ``` diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index b6bbfe690c1..a85f399a447 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -36,7 +36,7 @@ projects: Create the merge request [using the "Database reviewer" template](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/.gitlab/merge_request_templates/Database%20reviewer.md), adding your expertise your profile YAML file. Assign to a database maintainer or the -[Database Team's Engineering Manager](https://about.gitlab.com/handbook/engineering/development/enablement/database/). +[Database Team's Engineering Manager](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/). After the `team.yml` update is merged, the [Reviewer roulette](../code_review.md#reviewer-roulette) may recommend you as a database reviewer. @@ -53,17 +53,17 @@ that require a more in-depth discussion between the database reviewers and maint - [Database Office Hours Agenda](https://docs.google.com/document/d/1wgfmVL30F8SdMg-9yY6Y8djPSxWNvKmhR5XmsvYX1EI/edit). - <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [YouTube playlist with past recordings](https://www.youtube.com/playlist?list=PL05JrBw4t0Kp-kqXeiF7fF7cFYaKtdqXM). -You should also join the [#database-lab](../understanding_explain_plans.md#database-lab-engine) +You should also join the [#database-lab](understanding_explain_plans.md#database-lab-engine) Slack channel and get familiar with how to use Joe, the Slackbot that provides developers with their own clone of the production database. Understanding and efficiently using `EXPLAIN` plans is at the core of the database review process. The following guides provide a quick introduction and links to follow on more advanced topics: -- Guide on [understanding EXPLAIN plans](../understanding_explain_plans.md). +- Guide on [understanding EXPLAIN plans](understanding_explain_plans.md). - [Explaining the unexplainable series in `depesz`](https://www.depesz.com/tag/unexplainable/). -We also have licensed access to The Art of PostgreSQL available, if you are interested in getting access please check out the +We also have licensed access to The Art of PostgreSQL. If you are interested in getting access, check out the [issue (confidential)](https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/23). Finally, you can find various guides in the [Database guides](index.md) page that cover more specific @@ -95,7 +95,7 @@ are three times as likely to be picked by the [Danger bot](../dangerbot.md) as o ## What to do if you feel overwhelmed Similar to all types of reviews, [unblocking others is always a top priority](https://about.gitlab.com/handbook/values/#global-optimization). -Database reviewers are expected to [review assigned merge requests in a timely manner](../code_review.md#review-turnaround-time) +Database reviewers are expected to [review assigned merge requests in a timely manner](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-turnaround-time) or let the author know as soon as possible and help them find another reviewer or maintainer. We are doing reviews to help the rest of the GitLab team and, at the same time, get exposed diff --git a/doc/development/database/db_dump.md b/doc/development/database/db_dump.md new file mode 100644 index 00000000000..f2076cbc410 --- /dev/null +++ b/doc/development/database/db_dump.md @@ -0,0 +1,56 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Importing a database dump into a staging environment + +Sometimes it is useful to import the database from a production environment +into a staging environment for testing. The procedure below assumes you have +SSH and `sudo` access to both the production environment and the staging VM. + +**Destroy your staging VM** when you are done with it. It is important to avoid +data leaks. + +On the staging VM, add the following line to `/etc/gitlab/gitlab.rb` to speed up +large database imports. + +```shell +# On STAGING +echo "postgresql['checkpoint_segments'] = 64" | sudo tee -a /etc/gitlab/gitlab.rb +sudo touch /etc/gitlab/skip-auto-reconfigure +sudo gitlab-ctl reconfigure +sudo gitlab-ctl stop puma +sudo gitlab-ctl stop sidekiq +``` + +Next, we let the production environment stream a compressed SQL dump to our +local machine via SSH, and redirect this stream to a `psql` client on the staging +VM. + +```shell +# On LOCAL MACHINE +ssh -C gitlab.example.com sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_dump -Cc gitlabhq_production |\ + ssh -C staging-vm sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -d template1 +``` + +## Recreating directory structure + +If you need to re-create some directory structure on the staging server you can +use this procedure. + +First, on the production server, create a list of directories you want to +re-create. + +```shell +# On PRODUCTION +(umask 077; sudo find /var/opt/gitlab/git-data/repositories -maxdepth 1 -type d -print0 > directories.txt) +``` + +Copy `directories.txt` to the staging server and create the directories there. + +```shell +# On STAGING +sudo -u git xargs -0 mkdir -p < directories.txt +``` diff --git a/doc/development/database/filtering_by_label.md b/doc/development/database/filtering_by_label.md new file mode 100644 index 00000000000..29b0c75298e --- /dev/null +++ b/doc/development/database/filtering_by_label.md @@ -0,0 +1,179 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- +# Filtering by label + +## Introduction + +GitLab has [labels](../../user/project/labels.md) that can be assigned to issues, +merge requests, and epics. Labels on those objects are a many-to-many relation +through the polymorphic `label_links` table. + +To filter these objects by multiple labels - for instance, 'all open +issues with the label ~Plan and the label ~backend' - we generate a +query containing a `GROUP BY` clause. In a simple form, this looks like: + +```sql +SELECT + issues.* +FROM + issues + INNER JOIN label_links ON label_links.target_id = issues.id + AND label_links.target_type = 'Issue' + INNER JOIN labels ON labels.id = label_links.label_id +WHERE + issues.project_id = 13083 + AND (issues.state IN ('opened')) + AND labels.title IN ('Plan', + 'backend') +GROUP BY + issues.id +HAVING (COUNT(DISTINCT labels.title) = 2) +ORDER BY + issues.updated_at DESC, + issues.id DESC +LIMIT 20 OFFSET 0 +``` + +In particular, note that: + +1. We `GROUP BY issues.id` so that we can ... +1. Use the `HAVING (COUNT(DISTINCT labels.title) = 2)` condition to ensure that + all matched issues have both labels. + +This is more complicated than is ideal. It makes the query construction more +prone to errors (such as +[issue #15557](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15557)). + +## Attempt A: `WHERE EXISTS` + +### Attempt A1: use multiple subqueries with `WHERE EXISTS` + +In [issue #37137](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37137) +and its associated [merge request](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14022), +we tried to replace the `GROUP BY` with multiple uses of `WHERE EXISTS`. For the +example above, this would give: + +```sql +WHERE (EXISTS ( + SELECT + TRUE + FROM + label_links + INNER JOIN labels ON labels.id = label_links.label_id + WHERE + labels.title = 'Plan' + AND target_type = 'Issue' + AND target_id = issues.id)) +AND (EXISTS ( + SELECT + TRUE + FROM + label_links + INNER JOIN labels ON labels.id = label_links.label_id + WHERE + labels.title = 'backend' + AND target_type = 'Issue' + AND target_id = issues.id)) +``` + +While this worked without schema changes, and did improve readability somewhat, +it did not improve query performance. + +### Attempt A2: use label IDs in the `WHERE EXISTS` clause + +In [merge request #34503](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34503), we followed a similar approach to A1. But this time, we +did a separate query to fetch the IDs of the labels used in the filter so that we avoid the `JOIN` in the `EXISTS` clause and filter directly by +`label_links.label_id`. We also added a new index on `label_links` for the `target_id`, `label_id`, and `target_type` columns to speed up this query. + +Finding the label IDs wasn't straightforward because there could be multiple labels with the same title within a single root namespace. We solved +this by grouping the label IDs by title and then using the array of IDs in the `EXISTS` clauses. + +This resulted in a significant performance improvement. However, this optimization could not be applied to the dashboard pages +where we do not have a project or group context. We could not easily search for the label IDs here because that would mean searching across all +projects and groups that the user has access to. + +## Attempt B: Denormalize using an array column + +Having [removed MySQL support in GitLab 12.1](https://about.gitlab.com/blog/2019/06/27/removing-mysql-support/), +using [PostgreSQL's arrays](https://www.postgresql.org/docs/11/arrays.html) became more +tractable as we didn't have to support two databases. We discussed denormalizing +the `label_links` table for querying in +[issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651), +with two options: label IDs and titles. + +We can think of both of those as array columns on `issues`, `merge_requests`, +and `epics`: `issues.label_ids` would be an array column of label IDs, and +`issues.label_titles` would be an array of label titles. + +These array columns can be complemented with +[GIN indexes](https://www.postgresql.org/docs/11/gin-intro.html) to improve +matching. + +### Attempt B1: store label IDs for each object + +This has some strong advantages over titles: + +1. Unless a label is deleted, or a project is moved, we never need to + bulk-update the denormalized column. +1. It uses less storage than the titles. + +Unfortunately, our application design makes this hard. If we were able to query +just by label ID easily, we wouldn't need the `INNER JOIN labels` in the initial +query at the start of this document. GitLab allows users to filter by label +title across projects and even across groups, so a filter by the label ~Plan may +include labels with multiple distinct IDs. + +We do not want users to have to know about the different IDs, which means that +given this data set: + +| Project | ~Plan label ID | ~backend label ID | +| ------- | -------------- | ----------------- | +| A | 11 | 12 | +| B | 21 | 22 | +| C | 31 | 32 | + +We would need something like: + +```sql +WHERE + label_ids @> ARRAY[11, 12] + OR label_ids @> ARRAY[21, 22] + OR label_ids @> ARRAY[31, 32] +``` + +This can get even more complicated when we consider that in some cases, there +might be two ~backend labels - with different IDs - that could apply to the same +object, so the number of combinations would balloon further. + +### Attempt B2: store label titles for each object + +From the perspective of updating the object, this is the worst +option. We have to bulk update the objects when: + +1. The objects are moved from one project to another. +1. The project is moved from one group to another. +1. The label is renamed. +1. The label is deleted. + +It also uses much more storage. Querying is simple, though: + +```sql +WHERE + label_titles @> ARRAY['Plan', 'backend'] +``` + +And our [tests in issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651#note_188777346) +showed that this could be fast. + +However, at present, the disadvantages outweigh the advantages. + +## Conclusion + +We found a method A2 that does not need denormalization and improves the query performance significantly. This +did not apply to all cases, but we were able to apply method A1 to the rest of the cases so that we remove the +`GROUP BY` and `HAVING` clauses in all scenarios. + +This simplified the query and improved the performance in the most common cases. diff --git a/doc/development/database/foreign_keys.md b/doc/development/database/foreign_keys.md new file mode 100644 index 00000000000..7834e7d53c3 --- /dev/null +++ b/doc/development/database/foreign_keys.md @@ -0,0 +1,199 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Foreign Keys & Associations + +When adding an association to a model you must also add a foreign key. For +example, say you have the following model: + +```ruby +class User < ActiveRecord::Base + has_many :posts +end +``` + +Add a foreign key here on column `posts.user_id`. This ensures +that data consistency is enforced on database level. Foreign keys also mean that +the database can very quickly remove associated data (for example, when removing a +user), instead of Rails having to do this. + +## Adding Foreign Keys In Migrations + +Foreign keys can be added concurrently using `add_concurrent_foreign_key` as +defined in `Gitlab::Database::MigrationHelpers`. See the +[Migration Style Guide](../migration_style_guide.md) for more information. + +Keep in mind that you can only safely add foreign keys to existing tables after +you have removed any orphaned rows. The method `add_concurrent_foreign_key` +does not take care of this so you must do so manually. See +[adding foreign key constraint to an existing column](add_foreign_key_to_existing_column.md). + +## Updating Foreign Keys In Migrations + +Sometimes a foreign key constraint must be changed, preserving the column +but updating the constraint condition. For example, moving from +`ON DELETE CASCADE` to `ON DELETE SET NULL` or vice-versa. + +PostgreSQL does not prevent you from adding overlapping foreign keys. It +honors the most recently added constraint. This allows us to replace foreign keys without +ever losing foreign key protection on a column. + +To replace a foreign key: + +1. [Add the new foreign key without validation](add_foreign_key_to_existing_column.md#prevent-invalid-records) + + The name of the foreign key constraint must be changed to add a new + foreign key before removing the old one. + + ```ruby + class ReplaceFkOnPackagesPackagesProjectId < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + NEW_CONSTRAINT_NAME = 'fk_new' + + def up + add_concurrent_foreign_key(:packages_packages, :projects, column: :project_id, on_delete: :nullify, validate: false, name: NEW_CONSTRAINT_NAME) + end + + def down + with_lock_retries do + remove_foreign_key_if_exists(:packages_packages, column: :project_id, on_delete: :nullify, name: NEW_CONSTRAINT_NAME) + end + end + end + ``` + +1. [Validate the new foreign key](add_foreign_key_to_existing_column.md#validate-the-foreign-key) + + ```ruby + class ValidateFkNew < Gitlab::Database::Migration[2.0] + NEW_CONSTRAINT_NAME = 'fk_new' + + # foreign key added in <link to MR or path to migration adding new FK> + def up + validate_foreign_key(:packages_packages, name: NEW_CONSTRAINT_NAME) + end + + def down + # no-op + end + end + ``` + +1. Remove the old foreign key: + + ```ruby + class RemoveFkOld < Gitlab::Database::Migration[2.0] + OLD_CONSTRAINT_NAME = 'fk_old' + + # new foreign key added in <link to MR or path to migration adding new FK> + # and validated in <link to MR or path to migration validating new FK> + def up + remove_foreign_key_if_exists(:packages_packages, column: :project_id, on_delete: :cascade, name: OLD_CONSTRAINT_NAME) + end + + def down + # Validation is skipped here, so if rolled back, this will need to be revalidated in a separate migration + add_concurrent_foreign_key(:packages_packages, :projects, column: :project_id, on_delete: :cascade, validate: false, name: OLD_CONSTRAINT_NAME) + end + end + ``` + +## Cascading Deletes + +Every foreign key must define an `ON DELETE` clause, and in 99% of the cases +this should be set to `CASCADE`. + +## Indexes + +When adding a foreign key in PostgreSQL the column is not indexed automatically, +thus you must also add a concurrent index. Not doing so results in cascading +deletes being very slow. + +## Naming foreign keys + +By default Ruby on Rails uses the `_id` suffix for foreign keys. So we should +only use this suffix for associations between two tables. If you want to +reference an ID on a third party platform the `_xid` suffix is recommended. + +The spec `spec/db/schema_spec.rb` tests if all columns with the `_id` suffix +have a foreign key constraint. So if that spec fails, don't add the column to +`IGNORED_FK_COLUMNS`, but instead add the FK constraint, or consider naming it +differently. + +## Dependent Removals + +Don't define options such as `dependent: :destroy` or `dependent: :delete` when +defining an association. Defining these options means Rails handles the +removal of data, instead of letting the database handle this in the most +efficient way possible. + +In other words, this is bad and should be avoided at all costs: + +```ruby +class User < ActiveRecord::Base + has_many :posts, dependent: :destroy +end +``` + +Should you truly have a need for this it should be approved by a database +specialist first. + +You should also not define any `before_destroy` or `after_destroy` callbacks on +your models _unless_ absolutely required and only when approved by database +specialists. For example, if each row in a table has a corresponding file on a +file system it may be tempting to add a `after_destroy` hook. This however +introduces non database logic to a model, and means we can no longer rely on +foreign keys to remove the data as this would result in the file system data +being left behind. In such a case you should use a service class instead that +takes care of removing non database data. + +In cases where the relation spans multiple databases you have even +further problems using `dependent: :destroy` or the above hooks. You can +read more about alternatives at +[Avoid `dependent: :nullify` and `dependent: :destroy` across databases](multiple_databases.md#avoid-dependent-nullify-and-dependent-destroy-across-databases). + +## Alternative primary keys with `has_one` associations + +Sometimes a `has_one` association is used to create a one-to-one relationship: + +```ruby +class User < ActiveRecord::Base + has_one :user_config +end + +class UserConfig < ActiveRecord::Base + belongs_to :user +end +``` + +In these cases, there may be an opportunity to remove the unnecessary `id` +column on the associated table, `user_config.id` in this example. Instead, +the originating table ID can be used as the primary key for the associated +table: + +```ruby +create_table :user_configs, id: false do |t| + t.references :users, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade } + ... +end +``` + +Setting `default: nil` ensures a primary key sequence is not created, and since the primary key +automatically gets an index, we set `index: false` to avoid creating a duplicate. +You also need to add the new primary key to the model: + +```ruby +class UserConfig < ActiveRecord::Base + self.primary_key = :user_id + + belongs_to :user +end +``` + +Using a foreign key as primary key saves space but can make +[batch counting](../service_ping/implement.md#batch-counters) in [Service Ping](../service_ping/index.md) less efficient. +Consider using a regular `id` column if the table is relevant for Service Ping. diff --git a/doc/development/database/hash_indexes.md b/doc/development/database/hash_indexes.md new file mode 100644 index 00000000000..731639b6f06 --- /dev/null +++ b/doc/development/database/hash_indexes.md @@ -0,0 +1,26 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Hash Indexes + +PostgreSQL supports hash indexes besides the regular B-tree +indexes. Hash indexes however are to be avoided at all costs. While they may +_sometimes_ provide better performance the cost of rehashing can be very high. +More importantly: at least until PostgreSQL 10.0 hash indexes are not +WAL-logged, meaning they are not replicated to any replicas. From the PostgreSQL +documentation: + +> Hash index operations are not presently WAL-logged, so hash indexes might need +> to be rebuilt with REINDEX after a database crash if there were unwritten +> changes. Also, changes to hash indexes are not replicated over streaming or +> file-based replication after the initial base backup, so they give wrong +> answers to queries that subsequently use them. For these reasons, hash index +> use is presently discouraged. + +RuboCop is configured to register an offense when it detects the use of a hash +index. + +Instead of using hash indexes you should use regular B-tree indexes. diff --git a/doc/development/database/index.md b/doc/development/database/index.md index b427f54ff3c..8cf9a2eec04 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -16,52 +16,62 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Tooling -- [Understanding EXPLAIN plans](../understanding_explain_plans.md) +- [Understanding EXPLAIN plans](understanding_explain_plans.md) - [explain.depesz.com](https://explain.depesz.com/) or [explain.dalibo.com](https://explain.dalibo.com/) for visualizing the output of `EXPLAIN` - [pgFormatter](https://sqlformat.darold.net/) a PostgreSQL SQL syntax beautifier - [db:check-migrations job](dbcheck-migrations-job.md) ## Migrations +- [Different types of migrations](../migration_style_guide.md#choose-an-appropriate-migration-type) +- [Create a regular migration](../migration_style_guide.md#create-a-regular-schema-migration), including creating new models +- [Post-deployment migrations guidelines](post_deployment_migrations.md) and [how to create one](post_deployment_migrations.md#creating-migrations) +- [Background migrations guidelines](background_migrations.md) +- [Batched background migrations guidelines](batched_background_migrations.md) +- [Deleting migrations](deleting_migrations.md) +- [Running database migrations](database_debugging.md#migration-wrangling) - [Migrations for multiple databases](migrations_for_multiple_databases.md) - [Avoiding downtime in migrations](avoiding_downtime_in_migrations.md) -- [SQL guidelines](../sql.md) for working with SQL queries +- [When and how to write Rails migrations tests](../testing_guide/testing_migrations_guide.md) - [Migrations style guide](../migration_style_guide.md) for creating safe SQL migrations - [Testing Rails migrations](../testing_guide/testing_migrations_guide.md) guide - [Post deployment migrations](post_deployment_migrations.md) - [Background migrations](background_migrations.md) -- [Swapping tables](../swapping_tables.md) +- [Swapping tables](swapping_tables.md) - [Deleting migrations](deleting_migrations.md) +- [SQL guidelines](../sql.md) for working with SQL queries - [Partitioning tables](table_partitioning.md) ## Debugging -- Tracing the source of an SQL query using query comments with [Marginalia](../database_query_comments.md) +- [Resetting the database](database_debugging.md#delete-everything-and-start-over) +- [Accessing the database](database_debugging.md#manually-access-the-database) +- [Troubleshooting and debugging the database](database_debugging.md) +- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md) - Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) ## Best practices -- [Adding database indexes](../adding_database_indexes.md) -- [Foreign keys & associations](../foreign_keys.md) +- [Adding database indexes](adding_database_indexes.md) +- [Foreign keys & associations](foreign_keys.md) - [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md) - [`NOT NULL` constraints](not_null_constraints.md) - [Strings and the Text data type](strings_and_the_text_data_type.md) -- [Single table inheritance](../single_table_inheritance.md) -- [Polymorphic associations](../polymorphic_associations.md) -- [Serializing data](../serializing_data.md) -- [Hash indexes](../hash_indexes.md) -- [Storing SHA1 hashes as binary](../sha1_as_binary.md) -- [Iterating tables in batches](../iterating_tables_in_batches.md) -- [Insert into tables in batches](../insert_into_tables_in_batches.md) -- [Ordering table columns](../ordering_table_columns.md) -- [Verifying database capabilities](../verifying_database_capabilities.md) -- [Database Debugging and Troubleshooting](../database_debugging.md) -- [Query Count Limits](../query_count_limits.md) -- [Creating enums](../creating_enums.md) +- [Single table inheritance](single_table_inheritance.md) +- [Polymorphic associations](polymorphic_associations.md) +- [Serializing data](serializing_data.md) +- [Hash indexes](hash_indexes.md) +- [Storing SHA1 hashes as binary](sha1_as_binary.md) +- [Iterating tables in batches](iterating_tables_in_batches.md) +- [Insert into tables in batches](insert_into_tables_in_batches.md) +- [Ordering table columns](ordering_table_columns.md) +- [Verifying database capabilities](verifying_database_capabilities.md) +- [Query Count Limits](query_count_limits.md) +- [Creating enums](creating_enums.md) - [Client-side connection-pool](client_side_connection_pool.md) - [Updating multiple values](setting_multiple_values.md) - [Constraints naming conventions](constraint_naming_convention.md) -- [Query performance guidelines](../query_performance.md) +- [Query performance guidelines](query_performance.md) - [Pagination guidelines](pagination_guidelines.md) - [Pagination performance guidelines](pagination_performance_guidelines.md) - [Efficient `IN` operator queries](efficient_in_operator_queries.md) @@ -69,8 +79,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Case studies -- [Database case study: Filtering by label](../filtering_by_label.md) -- [Database case study: Namespaces storage statistics](../namespaces_storage_statistics.md) +- [Database case study: Filtering by label](filtering_by_label.md) +- [Database case study: Namespaces storage statistics](namespaces_storage_statistics.md) ## Miscellaneous diff --git a/doc/development/database/insert_into_tables_in_batches.md b/doc/development/database/insert_into_tables_in_batches.md new file mode 100644 index 00000000000..ebed3d16319 --- /dev/null +++ b/doc/development/database/insert_into_tables_in_batches.md @@ -0,0 +1,196 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +description: "Sometimes it is necessary to store large amounts of records at once, which can be inefficient +when iterating collections and performing individual `save`s. With the arrival of `insert_all` +in Rails 6, which operates at the row level (that is, using `Hash`es), GitLab has added a set +of APIs that make it safe and simple to insert ActiveRecord objects in bulk." +--- + +# Insert into tables in batches + +Sometimes it is necessary to store large amounts of records at once, which can be inefficient +when iterating collections and saving each record individually. With the arrival of +[`insert_all`](https://apidock.com/rails/ActiveRecord/Persistence/ClassMethods/insert_all) +in Rails 6, which operates at the row level (that is, using `Hash` objects), GitLab has added a set +of APIs that make it safe and simple to insert `ActiveRecord` objects in bulk. + +## Prepare `ApplicationRecord`s for bulk insertion + +In order for a model class to take advantage of the bulk insertion API, it has to include the +`BulkInsertSafe` concern first: + +```ruby +class MyModel < ApplicationRecord + # other includes here + # ... + include BulkInsertSafe # include this last + + # ... +end +``` + +The `BulkInsertSafe` concern has two functions: + +- It performs checks against your model class to ensure that it does not use ActiveRecord + APIs that are not safe to use with respect to bulk insertions (more on that below). +- It adds new class methods `bulk_insert!` and `bulk_upsert!`, which you can use to insert many records at once. + +## Insert records with `bulk_insert!` and `bulk_upsert!` + +If the target class passes the checks performed by `BulkInsertSafe`, you can insert an array of +ActiveRecord model objects as follows: + +```ruby +records = [MyModel.new, ...] + +MyModel.bulk_insert!(records) +``` + +Calls to `bulk_insert!` always attempt to insert _new records_. If instead +you would like to replace existing records with new values, while still inserting those +that do not already exist, then you can use `bulk_upsert!`: + +```ruby +records = [MyModel.new, existing_model, ...] + +MyModel.bulk_upsert!(records, unique_by: [:name]) +``` + +In this example, `unique_by` specifies the columns by which records are considered to be +unique and as such are updated if they existed prior to insertion. For example, if +`existing_model` has a `name` attribute, and if a record with the same `name` value already +exists, its fields are updated with those of `existing_model`. + +The `unique_by` parameter can also be passed as a `Symbol`, in which case it specifies +a database index by which a column is considered unique: + +```ruby +MyModel.bulk_insert!(records, unique_by: :index_on_name) +``` + +### Record validation + +The `bulk_insert!` method guarantees that `records` are inserted transactionally, and +runs validations on each record prior to insertion. If any record fails to validate, +an error is raised and the transaction is rolled back. You can turn off validations via +the `:validate` option: + +```ruby +MyModel.bulk_insert!(records, validate: false) +``` + +### Batch size configuration + +In those cases where the number of `records` is above a given threshold, insertions +occur in multiple batches. The default batch size is defined in +[`BulkInsertSafe::DEFAULT_BATCH_SIZE`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). +Assuming a default threshold of 500, inserting 950 records +would result in two batches being written sequentially (of size 500 and 450 respectively.) +You can override the default batch size via the `:batch_size` option: + +```ruby +MyModel.bulk_insert!(records, batch_size: 100) +``` + +Assuming the same number of 950 records, this would result in 10 batches being written instead. +Since this also affects the number of `INSERT` statements that occur, make sure you measure the +performance impact this might have on your code. There is a trade-off between the number of +`INSERT` statements the database has to process and the size and cost of each `INSERT`. + +### Handling duplicate records + +NOTE: +This parameter applies only to `bulk_insert!`. If you intend to update existing +records, use `bulk_upsert!` instead. + +It may happen that some records you are trying to insert already exist, which would result in +primary key conflicts. There are two ways to address this problem: failing fast by raising an +error or skipping duplicate records. The default behavior of `bulk_insert!` is to fail fast +and raise an `ActiveRecord::RecordNotUnique` error. + +If this is undesirable, you can instead skip duplicate records with the `skip_duplicates` flag: + +```ruby +MyModel.bulk_insert!(records, skip_duplicates: true) +``` + +### Requirements for safe bulk insertions + +Large parts of ActiveRecord's persistence API are built around the notion of callbacks. Many +of these callbacks fire in response to model life cycle events such as `save` or `create`. +These callbacks cannot be used with bulk insertions, since they are meant to be called for +every instance that is saved or created. Since these events do not fire when +records are inserted in bulk, we currently prevent their use. + +The specifics around which callbacks are explicitly allowed are defined in +[`BulkInsertSafe`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). +Consult the module source code for details. If your class uses callbacks that are not explicitly designated +safe and you `include BulkInsertSafe` the application fails with an error. + +### `BulkInsertSafe` versus `InsertAll` + +Internally, `BulkInsertSafe` is based on `InsertAll`, and you may wonder when to choose +the former over the latter. To help you make the decision, +the key differences between these classes are listed in the table below. + +| | Input type | Validates input | Specify batch size | Can bypass callbacks | Transactional | +|--------------- | -------------------- | --------------- | ------------------ | --------------------------------- | ------------- | +| `bulk_insert!` | ActiveRecord objects | Yes (optional) | Yes (optional) | No (prevents unsafe callback use) | Yes | +| `insert_all!` | Attribute hashes | No | No | Yes | Yes | + +To summarize, `BulkInsertSafe` moves bulk inserts closer to how ActiveRecord objects +and inserts would normally behave. However, if all you need is to insert raw data in bulk, then +`insert_all` is more efficient. + +## Insert `has_many` associations in bulk + +A common use case is to save collections of associated relations through the owner side of the relation, +where the owned relation is associated to the owner through the `has_many` class method: + +```ruby +owner = OwnerModel.new(owned_relations: array_of_owned_relations) +# saves all `owned_relations` one-by-one +owner.save! +``` + +This issues a single `INSERT`, and transaction, for every record in `owned_relations`, which is inefficient if +`array_of_owned_relations` is large. To remedy this, the `BulkInsertableAssociations` concern can be +used to declare that the owner defines associations that are safe for bulk insertion: + +```ruby +class OwnerModel < ApplicationRecord + # other includes here + # ... + include BulkInsertableAssociations # include this last + + has_many :my_models +end +``` + +Here `my_models` must be declared `BulkInsertSafe` (as described previously) for bulk insertions +to happen. You can now insert any yet unsaved records as follows: + +```ruby +BulkInsertableAssociations.with_bulk_insert do + owner = OwnerModel.new(my_models: array_of_my_model_instances) + # saves `my_models` using a single bulk insert (possibly via multiple batches) + owner.save! +end +``` + +You can still save relations that are not `BulkInsertSafe` in this block; they +simply are treated as if you had invoked `save` from outside the block. + +## Known limitations + +There are a few restrictions to how these APIs can be used: + +- `BulkInsertableAssociations`: + - It is currently only compatible with `has_many` relations. + - It does not yet support `has_many through: ...` relations. + +Moreover, input data should either be limited to around 1000 records at most, +or already batched prior to calling bulk insert. The `INSERT` statement runs in a single +transaction, so for large amounts of records it may negatively affect database stability. diff --git a/doc/development/database/iterating_tables_in_batches.md b/doc/development/database/iterating_tables_in_batches.md new file mode 100644 index 00000000000..6d7a57ecacb --- /dev/null +++ b/doc/development/database/iterating_tables_in_batches.md @@ -0,0 +1,598 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Iterating tables in batches + +Rails provides a method called `in_batches` that can be used to iterate over +rows in batches. For example: + +```ruby +User.in_batches(of: 10) do |relation| + relation.update_all(updated_at: Time.now) +end +``` + +Unfortunately, this method is implemented in a way that is not very efficient, +both query and memory usage wise. + +To work around this you can include the `EachBatch` module into your models, +then use the `each_batch` class method. For example: + +```ruby +class User < ActiveRecord::Base + include EachBatch +end + +User.each_batch(of: 10) do |relation| + relation.update_all(updated_at: Time.now) +end +``` + +This produces queries such as: + +```plaintext +User Load (0.7ms) SELECT "users"."id" FROM "users" WHERE ("users"."id" >= 41654) ORDER BY "users"."id" ASC LIMIT 1 OFFSET 1000 + (0.7ms) SELECT COUNT(*) FROM "users" WHERE ("users"."id" >= 41654) AND ("users"."id" < 42687) +``` + +The API of this method is similar to `in_batches`, though it doesn't support +all of the arguments that `in_batches` supports. You should always use +`each_batch` _unless_ you have a specific need for `in_batches`. + +## Iterating over non-unique columns + +One should proceed with extra caution. When you iterate over an attribute that is not unique, +even with the applied max batch size, there is no guarantee that the resulting batches do not +surpass it. The following snippet demonstrates this situation when one attempt to select +`Ci::Build` entries for users with `id` between `1` and `10,000`, the database returns +`1 215 178` matching rows. + +```ruby +[ gstg ] production> Ci::Build.where(user_id: (1..10_000)).size +=> 1215178 +``` + +This happens because the built relation is translated into the following query: + +```ruby +[ gstg ] production> puts Ci::Build.where(user_id: (1..10_000)).to_sql +SELECT "ci_builds".* FROM "ci_builds" WHERE "ci_builds"."type" = 'Ci::Build' AND "ci_builds"."user_id" BETWEEN 1 AND 10000 +=> nil +``` + +`And` queries which filter non-unique column by range `WHERE "ci_builds"."user_id" BETWEEN ? AND ?`, +even though the range size is limited to a certain threshold (`10,000` in the previous example) this +threshold does not translate to the size of the returned dataset. That happens because when taking +`n` possible values of attributes, one can't tell for sure that the number of records that contains +them is less than `n`. + +### Loose-index scan with `distinct_each_batch` + +When iterating over a non-unique column is necessary, use the `distinct_each_batch` helper +method. The helper uses the [loose-index scan technique](https://wiki.postgresql.org/wiki/Loose_indexscan) +(skip-index scan) to skip duplicated values within a database index. + +Example: iterating over distinct `author_id` in the Issue model + +```ruby +Issue.distinct_each_batch(column: :author_id, of: 1000) do |relation| + users = User.where(id: relation.select(:author_id)).to_a +end +``` + +The technique provides stable performance between the batches regardless of the data distribution. +The `relation` object returns an ActiveRecord scope where only the given `column` is available. +Other columns are not loaded. + +The underlying database queries use recursive CTEs, which adds extra overhead. We therefore advise to use +smaller batch sizes than those used for a standard `each_batch` iteration. + +## Column definition + +`EachBatch` uses the primary key of the model by default for the iteration. This works most of the +cases, however in some cases, you might want to use a different column for the iteration. + +```ruby +Project.distinct.each_batch(column: :creator_id, of: 10) do |relation| + puts User.where(id: relation.select(:creator_id)).map(&:id) +end +``` + +The query above iterates over the project creators and prints them out without duplications. + +NOTE: +In case the column is not unique (no unique index definition), calling the `distinct` method on +the relation is necessary. Using not unique column without `distinct` may result in `each_batch` +falling into an endless loop as described in following +[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/285097). + +## `EachBatch` in data migrations + +When dealing with data migrations the preferred way to iterate over a large volume of data is using +`EachBatch`. + +A special case of data migration is a [background migration](background_migrations.md#scheduling) +where the actual data modification is executed in a background job. The migration code that +determines the data ranges (slices) and schedules the background jobs uses `each_batch`. + +## Efficient usage of `each_batch` + +`EachBatch` helps to iterate over large tables. It's important to highlight that `EachBatch` +does not magically solve all iteration-related performance problems, and it might not help at +all in some scenarios. From the database point of view, correctly configured database indexes are +also necessary to make `EachBatch` perform well. + +### Example 1: Simple iteration + +Let's consider that we want to iterate over the `users` table and print the `User` records to the +standard output. The `users` table contains millions of records, thus running one query to fetch +the users likely times out. + + + +This is a simplified version of the `users` table which contains several rows. We have a few +smaller gaps in the `id` column to make the example a bit more realistic (a few records were +already deleted). Currently, we have one index on the `id` field. + +Loading all users into memory (avoid): + +```ruby +users = User.all + +users.each { |user| puts user.inspect } +``` + +Use `each_batch`: + +```ruby +# Note: for this example I picked 5 as the batch size, the default is 1_000 +User.each_batch(of: 5) do |relation| + relation.each { |user| puts user.inspect } +end +``` + +#### How `each_batch` works + +As the first step, it finds the lowest `id` (start `id`) in the table by executing the following +database query: + +```sql +SELECT "users"."id" FROM "users" ORDER BY "users"."id" ASC LIMIT 1 +``` + + + +Notice that the query only reads data from the index (`INDEX ONLY SCAN`), the table is not +accessed. Database indexes are sorted so taking out the first item is a very cheap operation. + +The next step is to find the next `id` (end `id`) which should respect the batch size +configuration. In this example we used a batch size of 5. `EachBatch` uses the `OFFSET` clause +to get a "shifted" `id` value. + +```sql +SELECT "users"."id" FROM "users" WHERE "users"."id" >= 1 ORDER BY "users"."id" ASC LIMIT 1 OFFSET 5 +``` + + + +Again, the query only looks into the index. The `OFFSET 5` takes out the sixth `id` value: this +query reads a maximum of six items from the index regardless of the table size or the iteration +count. + +At this point, we know the `id` range for the first batch. Now it's time to construct the query +for the `relation` block. + +```sql +SELECT "users".* FROM "users" WHERE "users"."id" >= 1 AND "users"."id" < 302 +``` + + + +Notice the `<` sign. Previously six items were read from the index and in this query, the last +value is "excluded". The query looks at the index to get the location of the five `user` +rows on the disk and read the rows from the table. The returned array is processed in Ruby. + +The first iteration is done. For the next iteration, the last `id` value is reused from the +previous iteration in order to find out the next end `id` value. + +```sql +SELECT "users"."id" FROM "users" WHERE "users"."id" >= 302 ORDER BY "users"."id" ASC LIMIT 1 OFFSET 5 +``` + + + +Now we can easily construct the `users` query for the second iteration. + +```sql +SELECT "users".* FROM "users" WHERE "users"."id" >= 302 AND "users"."id" < 353 +``` + + + +### Example 2: Iteration with filters + +Building on top of the previous example, we want to print users with zero sign-in count. We keep +track of the number of sign-ins in the `sign_in_count` column so we write the following code: + +```ruby +users = User.where(sign_in_count: 0) + +users.each_batch(of: 5) do |relation| + relation.each { |user| puts user.inspect } +end +``` + +`each_batch` produces the following SQL query for the start `id` value: + +```sql +SELECT "users"."id" FROM "users" WHERE "users"."sign_in_count" = 0 ORDER BY "users"."id" ASC LIMIT 1 +``` + +Selecting only the `id` column and ordering by `id` forces the database to use the +index on the `id` (primary key index) column however, we also have an extra condition on the +`sign_in_count` column. The column is not part of the index, so the database needs to look into +the actual table to find the first matching row. + + + +NOTE: +The number of scanned rows depends on the data distribution in the table. + +- Best case scenario: the first user was never logged in. The database reads only one row. +- Worst case scenario: all users were logged in at least once. The database reads all rows. + +In this particular example, the database had to read 10 rows (regardless of our batch size setting) +to determine the first `id` value. In a "real-world" application it's hard to predict whether the +filtering causes problems or not. In the case of GitLab, verifying the data on a +production replica is a good start, but keep in mind that data distribution on GitLab.com can be +different from self-managed instances. + +#### Improve filtering with `each_batch` + +##### Specialized conditional index + +```sql +CREATE INDEX index_on_users_never_logged_in ON users (id) WHERE sign_in_count = 0 +``` + +This is how our table and the newly created index looks like: + + + +This index definition covers the conditions on the `id` and `sign_in_count` columns thus makes the +`each_batch` queries very effective (similar to the simple iteration example). + +It's rare when a user was never signed in so we a anticipate small index size. Including only the +`id` in the index definition also helps to keep the index size small. + +##### Index on columns + +Later on, we might want to iterate over the table filtering for different `sign_in_count` values, in +those cases we cannot use the previously suggested conditional index because the `WHERE` condition +does not match with our new filter (`sign_in_count > 10`). + +To address this problem, we have two options: + +- Create another, conditional index to cover the new query. +- Replace the index with a more generalized configuration. + +NOTE: +Having multiple indexes on the same table and on the same columns could be a performance bottleneck +when writing data. + +Let's consider the following index (avoid): + +```sql +CREATE INDEX index_on_users_never_logged_in ON users (id, sign_in_count) +``` + +The index definition starts with the `id` column which makes the index very inefficient from data +selectivity point of view. + +```sql +SELECT "users"."id" FROM "users" WHERE "users"."sign_in_count" = 0 ORDER BY "users"."id" ASC LIMIT 1 +``` + +Executing the query above results in an `INDEX ONLY SCAN`. However, the query still needs to +iterate over an unknown number of entries in the index, and then find the first item where the +`sign_in_count` is `0`. + + + +We can improve the query significantly by swapping the columns in the index definition (prefer). + +```sql +CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count, id) +``` + + + +The following index definition does not work well with `each_batch` (avoid). + +```sql +CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count) +``` + +Since `each_batch` builds range queries based on the `id` column, this index cannot be used +efficiently. The DB reads the rows from the table or uses a bitmap search where the primary +key index is also read. + +##### "Slow" iteration + +Slow iteration means that we use a good index configuration to iterate over the table and +apply filtering on the yielded relation. + +```ruby +User.each_batch(of: 5) do |relation| + relation.where(sign_in_count: 0).each { |user| puts user inspect } +end +``` + +The iteration uses the primary key index (on the `id` column) which makes it safe from statement +timeouts. The filter (`sign_in_count: 0`) is applied on the `relation` where the `id` is already +constrained (range). The number of rows is limited. + +Slow iteration generally takes more time to finish. The iteration count is higher and +one iteration could yield fewer records than the batch size. Iterations may even yield +0 records. This is not an optimal solution; however, in some cases (especially when +dealing with large tables) this is the only viable option. + +### Using Subqueries + +Using subqueries in your `each_batch` query does not work well in most cases. Consider the following example: + +```ruby +projects = Project.where(creator_id: Issue.where(confidential: true).select(:author_id)) + +projects.each_batch do |relation| + # do something +end +``` + +The iteration uses the `id` column of the `projects` table. The batching does not affect the +subquery. This means for each iteration, the subquery is executed by the database. This adds a +constant "load" on the query which often ends up in statement timeouts. We have an unknown number +of [confidential issues](../../user/project/issues/confidential_issues.md), the execution time +and the accessed database rows depend on the data distribution in the `issues` table. + +NOTE: +Using subqueries works only when the subquery returns a small number of rows. + +#### Improving Subqueries + +When dealing with subqueries, a slow iteration approach could work: the filter on `creator_id` +can be part of the generated `relation` object. + +```ruby +projects = Project.all + +projects.each_batch do |relation| + relation.where(creator_id: Issue.where(confidential: true).select(:author_id)) +end +``` + +If the query on the `issues` table itself is not performant enough, a nested loop could be +constructed. Try to avoid it when possible. + +```ruby +projects = Project.all + +projects.each_batch do |relation| + issues = Issue.where(confidential: true) + + issues.each_batch do |issues_relation| + relation.where(creator_id: issues_relation.select(:author_id)) + end +end +``` + +If we know that the `issues` table has many more rows than `projects`, it would make sense to flip +the queries, where the `issues` table is batched first. + +### Using `JOIN` and `EXISTS` + +When to use `JOINS`: + +- When there's a 1:1 or 1:N relationship between the tables where we know that the joined record +(almost) always exists. This works well for "extension-like" tables: + - `projects` - `project_settings` + - `users` - `user_details` + - `users` - `user_statuses` +- `LEFT JOIN` works well in this case. Conditions on the joined table need to go to the yielded +relation so the iteration is not affected by the data distribution in the joined table. + +Example: + +```ruby +users = User.joins("LEFT JOIN personal_access_tokens on personal_access_tokens.user_id = users.id") + +users.each_batch do |relation| + relation.where("personal_access_tokens.name = 'name'") +end +``` + +`EXISTS` queries should be added only to the inner `relation` of the `each_batch` query: + +```ruby +User.each_batch do |relation| + relation.where("EXISTS (SELECT 1 FROM ...") +end +``` + +### Complex queries on the relation object + +When the `relation` object has several extra conditions, the execution plans might become +"unstable". + +Example: + +```ruby +Issue.each_batch do |relation| + relation + .joins(:metrics) + .joins(:merge_requests_closing_issues) + .where("id IN (SELECT ...)") + .where(confidential: true) +end +``` + +Here, we expect that the `relation` query reads the `BATCH_SIZE` of user records and then +filters down the results according to the provided queries. The planner might decide that +using a bitmap index lookup with the index on the `confidential` column is a better way to +execute the query. This can cause an unexpectedly high amount of rows to be read and the +query could time out. + +Problem: we know for sure that the relation is returning maximum `BATCH_SIZE` of records +however, the planner does not know this. + +Common table expression (CTE) trick to force the range query to execute first: + +```ruby +Issue.each_batch(of: 1000) do |relation| + cte = Gitlab::SQL::CTE.new(:batched_relation, relation.limit(1000)) + + scope = cte + .apply_to(Issue.all) + .joins(:metrics) + .joins(:merge_requests_closing_issues) + .where("id IN (SELECT ...)") + .where(confidential: true) + + puts scope.to_a +end +``` + +### `EachBatch` vs `BatchCount` + +When adding new counters for Service Ping, the preferred way to count records is using the +`Gitlab::Database::BatchCount` class. The iteration logic implemented in `BatchCount` +has similar performance characteristics like `EachBatch`. Most of the tips and suggestions +for improving `BatchCount` mentioned above applies to `BatchCount` as well. + +## Iterate with keyset pagination + +There are a few special cases where iterating with `EachBatch` does not work. `EachBatch` +requires one distinct column (usually the primary key), which makes the iteration impossible +for timestamp columns and tables with composite primary keys. + +Where `EachBatch` does not work, you can use +[keyset pagination](pagination_guidelines.md#keyset-pagination) to iterate over the +table or a range of rows. The scaling and performance characteristics are very similar to +`EachBatch`. + +Examples: + +- Iterate over the table in a specific order (timestamp columns) in combination with a tie-breaker +if column user to sort by does not contain unique values. +- Iterate over the table with composite primary keys. + +### Iterate over the issues in a project by creation date + +You can use keyset pagination to iterate over any database column in a specific order (for example, +`created_at DESC`). To ensure consistent order of the returned records with the same values for +`created_at`, use a tie-breaker column with unique values (for example, `id`). + +Assume you have the following index in the `issues` table: + +```sql +idx_issues_on_project_id_and_created_at_and_id" btree (project_id, created_at, id) +``` + +### Fetching records for further processing + +The following snippet iterates over issue records within the project using the specified order +(`created_at, id`). + +```ruby +scope = Issue.where(project_id: 278964).order(:created_at, :id) # id is the tie-breaker + +iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) + +iterator.each_batch(of: 100) do |records| + puts records.map(&:id) +end +``` + +You can add extra filters to the query. This example only lists the issue IDs created in the last +30 days: + +```ruby +scope = Issue.where(project_id: 278964).where('created_at > ?', 30.days.ago).order(:created_at, :id) # id is the tie-breaker + +iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) + +iterator.each_batch(of: 100) do |records| + puts records.map(&:id) +end +``` + +### Updating records in the batch + +For complex `ActiveRecord` queries, the `.update_all` method does not work well, because it +generates an incorrect `UPDATE` statement. +You can use raw SQL for updating records in batches: + +```ruby +scope = Issue.where(project_id: 278964).order(:created_at, :id) # id is the tie-breaker + +iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) + +iterator.each_batch(of: 100) do |records| + ApplicationRecord.connection.execute("UPDATE issues SET updated_at=NOW() WHERE issues.id in (#{records.dup.reselect(:id).to_sql})") +end +``` + +NOTE: +To keep the iteration stable and predictable, avoid updating the columns in the `ORDER BY` clause. + +### Iterate over the `merge_request_diff_commits` table + +The `merge_request_diff_commits` table uses a composite primary key (`merge_request_diff_id, +relative_order`), which makes `EachBatch` impossible to use efficiently. + +To paginate over the `merge_request_diff_commits` table, you can use the following snippet: + +```ruby +# Custom order object configuration: +order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'merge_request_diff_id', + order_expression: MergeRequestDiffCommit.arel_table[:merge_request_diff_id].asc, + nullable: :not_nullable, + distinct: false, + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'relative_order', + order_expression: MergeRequestDiffCommit.arel_table[:relative_order].asc, + nullable: :not_nullable, + distinct: false, + ) +]) +MergeRequestDiffCommit.include(FromUnion) # keyset pagination generates UNION queries + +scope = MergeRequestDiffCommit.order(order) + +iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) + +iterator.each_batch(of: 100) do |records| + puts records.map { |record| [record.merge_request_diff_id, record.relative_order] }.inspect +end +``` + +### Order object configuration + +Keyset pagination works well with simple `ActiveRecord` `order` scopes +([first example](#iterate-over-the-issues-in-a-project-by-creation-date). +However, in special cases, you need to describe the columns in the `ORDER BY` clause (second example) +for the underlying keyset pagination library. When the `ORDER BY` configuration cannot be +automatically determined by the keyset pagination library, an error is raised. + +The code comments of the +[`Gitlab::Pagination::Keyset::Order`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/pagination/keyset/order.rb) +and [`Gitlab::Pagination::Keyset::ColumnOrderDefinition`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/pagination/keyset/column_order_definition.rb) +classes give an overview of the possible options for configuring the `ORDER BY` clause. You can +also find a few code examples in the +[keyset pagination](keyset_pagination.md#complex-order-configuration) documentation. diff --git a/doc/development/database/loose_foreign_keys.md b/doc/development/database/loose_foreign_keys.md index 6aa1b9c40ff..8dbccf048d7 100644 --- a/doc/development/database/loose_foreign_keys.md +++ b/doc/development/database/loose_foreign_keys.md @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w In relational databases (including PostgreSQL), foreign keys provide a way to link two database tables together, and ensure data-consistency between them. In GitLab, -[foreign keys](../foreign_keys.md) are vital part of the database design process. +[foreign keys](foreign_keys.md) are vital part of the database design process. Most of our database tables have foreign keys. With the ongoing database [decomposition work](https://gitlab.com/groups/gitlab-org/-/epics/6168), @@ -221,8 +221,8 @@ ON DELETE CASCADE; ``` The migration must run after the `DELETE` trigger is installed and the loose -foreign key definition is deployed. As such, it must be a [post-deployment -migration](post_deployment_migrations.md) dated after the migration for the +foreign key definition is deployed. As such, it must be a +[post-deployment migration](post_deployment_migrations.md) dated after the migration for the trigger. If the foreign key is deleted earlier, there is a good chance of introducing data inconsistency which needs manual cleanup: @@ -251,8 +251,8 @@ When the loose foreign key definition is no longer needed (parent table is remov we need to remove the definition from the YAML file and ensure that we don't leave pending deleted records in the database. -1. Remove the loose foreign key definition from the configuration (`config/gitlab_loose_foreign_keys.yml`). 1. Remove the deletion tracking trigger from the parent table (if the parent table is still there). +1. Remove the loose foreign key definition from the configuration (`config/gitlab_loose_foreign_keys.yml`). 1. Remove leftover deleted records from the `loose_foreign_keys_deleted_records` table. Migration for removing the trigger: @@ -395,8 +395,7 @@ We considered using these Rails features as an alternative to foreign keys but t For non-trivial objects that need to clean up data outside the database (for example, object storage) where you might wish to use `dependent: :destroy`, see alternatives in -[Avoid `dependent: :nullify` and `dependent: :destroy` across -databases](./multiple_databases.md#avoid-dependent-nullify-and-dependent-destroy-across-databases). +[Avoid `dependent: :nullify` and `dependent: :destroy` across databases](multiple_databases.md#avoid-dependent-nullify-and-dependent-destroy-across-databases). ## Risks of loose foreign keys and possible mitigations diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index 9641ea37002..31fc454f8a7 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -1,15 +1,14 @@ --- stage: Data Stores -group: Sharding +group: Pods info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- # Multiple Databases To allow GitLab to scale further we -[decomposed the GitLab application database into multiple -databases](https://gitlab.com/groups/gitlab-org/-/epics/6168). The two databases -are `main` and `ci`. GitLab supports being run with either one database or two databases. +[decomposed the GitLab application database into multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6168). +The two databases are `main` and `ci`. GitLab supports being run with either one database or two databases. On GitLab.com we are using two separate databases. ## GitLab Schema @@ -246,7 +245,7 @@ where projects_with_ci_feature_usage.ci_feature = 'code_coverage' ``` The above example uses as a text column for simplicity but we should probably -use an [enum](../creating_enums.md) to save space. +use an [enum](creating_enums.md) to save space. The downside of this new design is that this may need to be updated (removed if the `ci_daily_build_group_report_results` is deleted). diff --git a/doc/development/database/namespaces_storage_statistics.md b/doc/development/database/namespaces_storage_statistics.md new file mode 100644 index 00000000000..702129b9ddb --- /dev/null +++ b/doc/development/database/namespaces_storage_statistics.md @@ -0,0 +1,193 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Database case study: Namespaces storage statistics + +## Introduction + +On [Storage and limits management for groups](https://gitlab.com/groups/gitlab-org/-/epics/886), +we want to facilitate a method for easily viewing the amount of +storage consumed by a group, and allow easy management. + +## Proposal + +1. Create a new ActiveRecord model to hold the namespaces' statistics in an aggregated form (only for root [namespaces](../../user/namespace/index.md)). +1. Refresh the statistics in this model every time a project belonging to this namespace is changed. + +## Problem + +In GitLab, we update the project storage statistics through a +[callback](https://gitlab.com/gitlab-org/gitlab/-/blob/4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e/app/models/project.rb#L97) +every time the project is saved. + +The summary of those statistics per namespace is then retrieved +by [`Namespaces#with_statistics`](https://gitlab.com/gitlab-org/gitlab/-/blob/4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e/app/models/namespace.rb#L70) scope. Analyzing this query we noticed that: + +- It takes up to `1.2` seconds for namespaces with over `15k` projects. +- It can't be analyzed with [ChatOps](../chatops_on_gitlabcom.md), as it times out. + +Additionally, the pattern that is currently used to update the project statistics +(the callback) doesn't scale adequately. It is currently one of the largest +[database queries transactions on production](https://gitlab.com/gitlab-org/gitlab/-/issues/29070) +that takes the most time overall. We can't add one more query to it as +it increases the transaction's length. + +Because of all of the above, we can't apply the same pattern to store +and update the namespaces statistics, as the `namespaces` table is one +of the largest tables on GitLab.com. Therefore we needed to find a performant and +alternative method. + +## Attempts + +### Attempt A: PostgreSQL materialized view + +Model can be updated through a refresh strategy based on a project routes SQL and a [materialized view](https://www.postgresql.org/docs/11/rules-materializedviews.html): + +```sql +SELECT split_part("rs".path, '/', 1) as root_path, + COALESCE(SUM(ps.storage_size), 0) AS storage_size, + COALESCE(SUM(ps.repository_size), 0) AS repository_size, + COALESCE(SUM(ps.wiki_size), 0) AS wiki_size, + COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size, + COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size, + COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size, + COALESCE(SUM(ps.packages_size), 0) AS packages_size, + COALESCE(SUM(ps.snippets_size), 0) AS snippets_size, + COALESCE(SUM(ps.uploads_size), 0) AS uploads_size +FROM "projects" + INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' + INNER JOIN project_statistics ps ON ps.project_id = projects.id +GROUP BY root_path +``` + +We could then execute the query with: + +```sql +REFRESH MATERIALIZED VIEW root_namespace_storage_statistics; +``` + +While this implied a single query update (and probably a fast one), it has some downsides: + +- Materialized views syntax varies from PostgreSQL and MySQL. While this feature was worked on, MySQL was still supported by GitLab. +- Rails does not have native support for materialized views. We'd need to use a specialized gem to take care of the management of the database views, which implies additional work. + +### Attempt B: An update through a CTE + +Similar to Attempt A: Model update done through a refresh strategy with a [Common Table Expression](https://www.postgresql.org/docs/9.1/queries-with.html) + +```sql +WITH refresh AS ( + SELECT split_part("rs".path, '/', 1) as root_path, + COALESCE(SUM(ps.storage_size), 0) AS storage_size, + COALESCE(SUM(ps.repository_size), 0) AS repository_size, + COALESCE(SUM(ps.wiki_size), 0) AS wiki_size, + COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size, + COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size, + COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size, + COALESCE(SUM(ps.packages_size), 0) AS packages_size, + COALESCE(SUM(ps.snippets_size), 0) AS snippets_size, + COALESCE(SUM(ps.uploads_size), 0) AS uploads_size + FROM "projects" + INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' + INNER JOIN project_statistics ps ON ps.project_id = projects.id + GROUP BY root_path) +UPDATE namespace_storage_statistics +SET storage_size = refresh.storage_size, + repository_size = refresh.repository_size, + wiki_size = refresh.wiki_size, + lfs_objects_size = refresh.lfs_objects_size, + build_artifacts_size = refresh.build_artifacts_size, + pipeline_artifacts_size = refresh.pipeline_artifacts_size, + packages_size = refresh.packages_size, + snippets_size = refresh.snippets_size, + uploads_size = refresh.uploads_size +FROM refresh + INNER JOIN routes rs ON rs.path = refresh.root_path AND rs.source_type = 'Namespace' +WHERE namespace_storage_statistics.namespace_id = rs.source_id +``` + +Same benefits and downsides as attempt A. + +### Attempt C: Get rid of the model and store the statistics on Redis + +We could get rid of the model that stores the statistics in aggregated form and instead use a Redis Set. +This would be the [boring solution](https://about.gitlab.com/handbook/values/#boring-solutions) and the fastest one +to implement, as GitLab already includes Redis as part of its [Architecture](../architecture.md#redis). + +The downside of this approach is that Redis does not provide the same persistence/consistency guarantees as PostgreSQL, +and this is information we can't afford to lose in a Redis failure. + +### Attempt D: Tag the root namespace and its child namespaces + +Directly relate the root namespace to its child namespaces, so +whenever a namespace is created without a parent, this one is tagged +with the root namespace ID: + +| ID | root ID | parent ID | +|:---|:--------|:----------| +| 1 | 1 | NULL | +| 2 | 1 | 1 | +| 3 | 1 | 2 | + +To aggregate the statistics inside a namespace we'd execute something like: + +```sql +SELECT COUNT(...) +FROM projects +WHERE namespace_id IN ( + SELECT id + FROM namespaces + WHERE root_id = X +) +``` + +Even though this approach would make aggregating much easier, it has some major downsides: + +- We'd have to migrate **all namespaces** by adding and filling a new column. Because of the size of the table, dealing with time/cost would be significant. The background migration would take approximately `153h`, see <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29772>. +- Background migration has to be shipped one release before, delaying the functionality by another milestone. + +### Attempt E (final): Update the namespace storage statistics asynchronously + +This approach consists of continuing to use the incremental statistics updates we already have, +but we refresh them through Sidekiq jobs and in different transactions: + +1. Create a second table (`namespace_aggregation_schedules`) with two columns `id` and `namespace_id`. +1. Whenever the statistics of a project changes, insert a row into `namespace_aggregation_schedules` + - We don't insert a new row if there's already one related to the root namespace. + - Keeping in mind the length of the transaction that involves updating `project_statistics`(<https://gitlab.com/gitlab-org/gitlab/-/issues/29070>), the insertion should be done in a different transaction and through a Sidekiq Job. +1. After inserting the row, we schedule another worker to be executed asynchronously at two different moments: + - One enqueued for immediate execution and another one scheduled in `1.5h` hours. + - We only schedule the jobs, if we can obtain a `1.5h` lease on Redis on a key based on the root namespace ID. + - If we can't obtain the lease, it indicates there's another aggregation already in progress, or scheduled in no more than `1.5h`. +1. This worker will: + - Update the root namespace storage statistics by querying all the namespaces through a service. + - Delete the related `namespace_aggregation_schedules` after the update. +1. Another Sidekiq job is also included to traverse any remaining rows on the `namespace_aggregation_schedules` table and schedule jobs for every pending row. + - This job is scheduled with cron to run every night (UTC). + +This implementation has the following benefits: + +- All the updates are done asynchronously, so we're not increasing the length of the transactions for `project_statistics`. +- We're doing the update in a single SQL query. +- It is compatible with PostgreSQL and MySQL. +- No background migration required. + +The only downside of this approach is that namespaces' statistics are updated up to `1.5` hours after the change is done, +which means there's a time window in which the statistics are inaccurate. Because we're still not +[enforcing storage limits](https://gitlab.com/gitlab-org/gitlab/-/issues/17664), this is not a major problem. + +## Conclusion + +Updating the storage statistics asynchronously, was the less problematic and +performant approach of aggregating the root namespaces. + +All the details regarding this use case can be found on: + +- <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62214> +- Merge Request with the implementation: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28996> + +Performance of the namespace storage statistics were measured in staging and production (GitLab.com). All results were posted +on <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64092>: No problem has been reported so far. diff --git a/doc/development/database/not_null_constraints.md b/doc/development/database/not_null_constraints.md index 3962307f80d..9b3d017b09f 100644 --- a/doc/development/database/not_null_constraints.md +++ b/doc/development/database/not_null_constraints.md @@ -135,7 +135,7 @@ post-deployment migration or a background data migration: - If the data volume is less than `1000` records, then the data migration can be executed within the post-migration. - If the data volume is higher than `1000` records, it's advised to create a background migration. -When unsure about which option to use, please contact the Database team for advice. +When unsure about which option to use, contact the Database team for advice. Back to our example, the epics table is not considerably large nor frequently accessed, so we add a post-deployment migration for the 13.0 milestone (current), @@ -206,6 +206,6 @@ In that rare case you need 3 releases end-to-end: 1. Release `N.M+1` - Cleanup the background migration. 1. Release `N.M+2` - Validate the `NOT NULL` constraint. -For these cases, please consult the database team early in the update cycle. The `NOT NULL` +For these cases, consult the database team early in the update cycle. The `NOT NULL` constraint may not be required or other options could exist that do not affect really large or frequently accessed tables. diff --git a/doc/development/database/ordering_table_columns.md b/doc/development/database/ordering_table_columns.md new file mode 100644 index 00000000000..7cd3d4fb208 --- /dev/null +++ b/doc/development/database/ordering_table_columns.md @@ -0,0 +1,152 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Ordering Table Columns in PostgreSQL + +For GitLab we require that columns of new tables are ordered to use the +least amount of space. An easy way of doing this is to order them based on the +type size in descending order with variable sizes (`text`, `varchar`, arrays, +`json`, `jsonb`, and so on) at the end. + +Similar to C structures the space of a table is influenced by the order of +columns. This is because the size of columns is aligned depending on the type of +the following column. Let's consider an example: + +- `id` (integer, 4 bytes) +- `name` (text, variable) +- `user_id` (integer, 4 bytes) + +The first column is a 4-byte integer. The next is text of variable length. The +`text` data type requires 1-word alignment, and on 64-bit platform, 1 word is 8 +bytes. To meet the alignment requirements, four zeros are to be added right +after the first column, so `id` occupies 4 bytes, then 4 bytes of alignment +padding, and only next `name` is being stored. Therefore, in this case, 8 bytes +are spent for storing a 4-byte integer. + +The space between rows is also subject to alignment padding. The `user_id` +column takes only 4 bytes, and on 64-bit platform, 4 zeroes are added for +alignment padding, to allow storing the next row beginning with the "clear" word. + +As a result, the actual size of each column would be (omitting variable length +data and 24-byte tuple header): 8 bytes, variable, 8 bytes. This means that +each row requires at least 16 bytes for the two 4-byte integers. If a table +has a few rows this is not an issue. However, once you start storing millions of +rows you can save space by using a different order. For the above example, the +ideal column order would be the following: + +- `id` (integer, 4 bytes) +- `user_id` (integer, 4 bytes) +- `name` (text, variable) + +or + +- `name` (text, variable) +- `id` (integer, 4 bytes) +- `user_id` (integer, 4 bytes) + +In these examples, the `id` and `user_id` columns are packed together, which +means we only need 8 bytes to store _both_ of them. This in turn means each row +requires 8 bytes less space. + +Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. +We are using `integer` in the examples to showcase a more realistic reordering scenario. + +## Type Sizes + +While the [PostgreSQL documentation](https://www.postgresql.org/docs/current/datatype.html) contains plenty +of information we list the sizes of common types here so it's easier to +look them up. Here "word" refers to the word size, which is 4 bytes for a 32 +bits platform and 8 bytes for a 64 bits platform. + +| Type | Size | Alignment needed | +|:-----------------|:-------------------------------------|:-----------| +| `smallint` | 2 bytes | 1 word | +| `integer` | 4 bytes | 1 word | +| `bigint` | 8 bytes | 8 bytes | +| `real` | 4 bytes | 1 word | +| `double precision` | 8 bytes | 8 bytes | +| `boolean` | 1 byte | not needed | +| `text` / `string` | variable, 1 byte plus the data | 1 word | +| `bytea` | variable, 1 or 4 bytes plus the data | 1 word | +| `timestamp` | 8 bytes | 8 bytes | +| `timestamptz` | 8 bytes | 8 bytes | +| `date` | 4 bytes | 1 word | + +A "variable" size means the actual size depends on the value being stored. If +PostgreSQL determines this can be embedded directly into a row it may do so, but +for very large values it stores the data externally and store a pointer (of +1 word in size) in the column. Because of this variable sized columns should +always be at the end of a table. + +## Real Example + +Let's use the `events` table as an example, which currently has the following +layout: + +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `target_id` | integer | 4 bytes | +| `title` | character varying | variable | +| `data` | text | variable | +| `project_id` | integer | 4 bytes | +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | + +After adding padding to align the columns this would translate to columns being +divided into fixed size chunks as follows: + +| Chunk Size | Columns | +|:-----------|:----------------------| +| 8 bytes | `id` | +| variable | `target_type` | +| 8 bytes | `target_id` | +| variable | `title` | +| variable | `data` | +| 8 bytes | `project_id` | +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `action`, `author_id` | + +This means that excluding the variable sized data and tuple header, we need at +least 8 * 6 = 48 bytes per row. + +We can optimise this by using the following column order instead: + +| Column | Type | Size | +|:--------------|:----------------------------|:---------| +| `created_at` | timestamp without time zone | 8 bytes | +| `updated_at` | timestamp without time zone | 8 bytes | +| `id` | integer | 4 bytes | +| `target_id` | integer | 4 bytes | +| `project_id` | integer | 4 bytes | +| `action` | integer | 4 bytes | +| `author_id` | integer | 4 bytes | +| `target_type` | character varying | variable | +| `title` | character varying | variable | +| `data` | text | variable | + +This would produce the following chunks: + +| Chunk Size | Columns | +|:-----------|:-----------------------| +| 8 bytes | `created_at` | +| 8 bytes | `updated_at` | +| 8 bytes | `id`, `target_id` | +| 8 bytes | `project_id`, `action` | +| 8 bytes | `author_id` | +| variable | `target_type` | +| variable | `title` | +| variable | `data` | + +Here we only need 40 bytes per row excluding the variable sized data and 24-byte +tuple header. 8 bytes being saved may not sound like much, but for tables as +large as the `events` table it does begin to matter. For example, when storing +80 000 000 rows this translates to a space saving of at least 610 MB, all by +just changing the order of a few columns. diff --git a/doc/development/database/pagination_guidelines.md b/doc/development/database/pagination_guidelines.md index 1641708ce01..fe2e3b46939 100644 --- a/doc/development/database/pagination_guidelines.md +++ b/doc/development/database/pagination_guidelines.md @@ -192,7 +192,7 @@ The query execution plan shows that this query is efficient, the database only r (6 rows) ``` -See the [Understanding EXPLAIN plans](../understanding_explain_plans.md) to find more information about reading execution plans. +See the [Understanding EXPLAIN plans](understanding_explain_plans.md) to find more information about reading execution plans. Let's visit the 50_000th page: diff --git a/doc/development/database/pagination_performance_guidelines.md b/doc/development/database/pagination_performance_guidelines.md index b5040e499e4..0fef246f133 100644 --- a/doc/development/database/pagination_performance_guidelines.md +++ b/doc/development/database/pagination_performance_guidelines.md @@ -12,11 +12,11 @@ The following document gives a few ideas for improving the pagination (sorting) When ordering the columns it's advised to order by distinct columns only. Consider the following example: -|`id`|`created_at`| -|-|-| -|1|2021-01-04 14:13:43| -|2|2021-01-05 19:03:12| -|3|2021-01-05 19:03:12| +| `id` | `created_at` | +|------|-----------------------| +| `1` | `2021-01-04 14:13:43` | +| `2` | `2021-01-05 19:03:12` | +| `3` | `2021-01-05 19:03:12` | If we order by `created_at`, the result would likely depend on how the records are located on the disk. diff --git a/doc/development/database/polymorphic_associations.md b/doc/development/database/polymorphic_associations.md new file mode 100644 index 00000000000..ac4dc7720a5 --- /dev/null +++ b/doc/development/database/polymorphic_associations.md @@ -0,0 +1,152 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Polymorphic Associations + +**Summary:** always use separate tables instead of polymorphic associations. + +Rails makes it possible to define so called "polymorphic associations". This +usually works by adding two columns to a table: a target type column, and a +target ID. For example, at the time of writing we have such a setup for +`members` with the following columns: + +- `source_type`: a string defining the model to use, can be either `Project` or + `Namespace`. +- `source_id`: the ID of the row to retrieve based on `source_type`. For + example, when `source_type` is `Project` then `source_id` contains a + project ID. + +While such a setup may appear to be useful, it comes with many drawbacks; enough +that you should avoid this at all costs. + +## Space Wasted + +Because this setup relies on string values to determine the model to use, it +wastes a lot of space. For example, for `Project` and `Namespace` the +maximum size is 9 bytes, plus 1 extra byte for every string when using +PostgreSQL. While this may only be 10 bytes per row, given enough tables and +rows using such a setup we can end up wasting quite a bit of disk space and +memory (for any indexes). + +## Indexes + +Because our associations are broken up into two columns this may result in +requiring composite indexes for queries to be performed efficiently. While +composite indexes are not wrong at all, they can be tricky to set up as the +ordering of columns in these indexes is important to ensure optimal performance. + +## Consistency + +One really big problem with polymorphic associations is being unable to enforce +data consistency on the database level using foreign keys. For consistency to be +enforced on the database level one would have to write their own foreign key +logic to support polymorphic associations. + +Enforcing consistency on the database level is absolutely crucial for +maintaining a healthy environment, and thus is another reason to avoid +polymorphic associations. + +## Query Overhead + +When using polymorphic associations you always need to filter using both +columns. For example, you may end up writing a query like this: + +```sql +SELECT * +FROM members +WHERE source_type = 'Project' +AND source_id = 13083; +``` + +Here PostgreSQL can perform the query quite efficiently if both columns are +indexed. As the query gets more complex, it may not be able to use these +indexes effectively. + +## Mixed Responsibilities + +Similar to functions and classes, a table should have a single responsibility: +storing data with a certain set of pre-defined columns. When using polymorphic +associations, you are storing different types of data (possibly with +different columns set) in the same table. + +## The Solution + +Fortunately, there is a solution to these problems: use a +separate table for every type you would otherwise store in the same table. Using +a separate table allows you to use everything a database may provide to ensure +consistency and query data efficiently, without any additional application logic +being necessary. + +Let's say you have a `members` table storing both approved and pending members, +for both projects and groups, and the pending state is determined by the column +`requested_at` being set or not. Schema wise such a setup can lead to various +columns only being set for certain rows, wasting space. It's also possible that +certain indexes are only set for certain rows, again wasting space. Finally, +querying such a table requires less than ideal queries. For example: + +```sql +SELECT * +FROM members +WHERE requested_at IS NULL +AND source_type = 'GroupMember' +AND source_id = 4 +``` + +Instead such a table should be broken up into separate tables. For example, you +may end up with 4 tables in this case: + +- project_members +- group_members +- pending_project_members +- pending_group_members + +This makes querying data trivial. For example, to get the members of a group +you'd run: + +```sql +SELECT * +FROM group_members +WHERE group_id = 4 +``` + +To get all the pending members of a group in turn you'd run: + +```sql +SELECT * +FROM pending_group_members +WHERE group_id = 4 +``` + +If you want to get both you can use a `UNION`, though you need to be explicit +about what columns you want to `SELECT` as otherwise the result set uses the +columns of the first query. For example: + +```sql +SELECT id, 'Group' AS target_type, group_id AS target_id +FROM group_members + +UNION ALL + +SELECT id, 'Project' AS target_type, project_id AS target_id +FROM project_members +``` + +The above example is perhaps a bit silly, but it shows that there's nothing +stopping you from merging the data together and presenting it on the same page. +Selecting columns explicitly can also speed up queries as the database has to do +less work to get the data (compared to selecting all columns, even ones you're +not using). + +Our schema also becomes easier. No longer do we need to both store and index the +`source_type` column, we can define foreign keys easily, and we don't need to +filter rows using the `IS NULL` condition. + +To summarize: using separate tables allows us to use foreign keys effectively, +create indexes only where necessary, conserve space, query data more +efficiently, and scale these tables more easily (for example, by storing them on +separate disks). A nice side effect of this is that code can also become easier, +as a single model isn't responsible for handling different kinds of +data. diff --git a/doc/development/database/post_deployment_migrations.md b/doc/development/database/post_deployment_migrations.md index a49c77ca047..8166fcc8905 100644 --- a/doc/development/database/post_deployment_migrations.md +++ b/doc/development/database/post_deployment_migrations.md @@ -25,6 +25,10 @@ This however skips post deployment migrations: SKIP_POST_DEPLOYMENT_MIGRATIONS=true bundle exec rake db:migrate ``` +For GitLab.com, these migrations are executed on a daily basis at the discretion of +release managers through the +[post-deploy migration pipeline](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md). + ## Deployment Integration Say you're using Chef for deploying new versions of GitLab and you'd like to run diff --git a/doc/development/database/query_count_limits.md b/doc/development/database/query_count_limits.md new file mode 100644 index 00000000000..a888bbfc6e7 --- /dev/null +++ b/doc/development/database/query_count_limits.md @@ -0,0 +1,70 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Query Count Limits + +Each controller or API endpoint is allowed to execute up to 100 SQL queries and +in test environments we raise an error when this threshold is exceeded. + +## Solving Failing Tests + +When a test fails because it executes more than 100 SQL queries there are two +solutions to this problem: + +- Reduce the number of SQL queries that are executed. +- Disable query limiting for the controller or API endpoint. + +You should only resort to disabling query limits when an existing controller or endpoint +is to blame as in this case reducing the number of SQL queries can take a lot of +effort. Newly added controllers and endpoints are not allowed to execute more +than 100 SQL queries and no exceptions are made for this rule. _If_ a large +number of SQL queries is necessary to perform certain work it's best to have +this work performed by Sidekiq instead of doing this directly in a web request. + +## Disable query limiting + +In the event that you _have_ to disable query limits for a controller, you must first +create an issue. This issue should (preferably in the title) mention the +controller or endpoint and include the appropriate labels (`database`, +`performance`, and at least a team specific label such as `Discussion`). + +After the issue has been created, you can disable query limits on the code in question. For +Rails controllers it's best to create a `before_action` hook that runs as early +as possible. The called method in turn should call +`Gitlab::QueryLimiting.disable!('issue URL here')`. For example: + +```ruby +class MyController < ApplicationController + before_action :disable_query_limiting, only: [:show] + + def index + # ... + end + + def show + # ... + end + + def disable_query_limiting + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/...') + end +end +``` + +By using a `before_action` you don't have to modify the controller method in +question, reducing the likelihood of merge conflicts. + +For Grape API endpoints there unfortunately is not a reliable way of running a +hook before a specific endpoint. This means that you have to add the allowlist +call directly into the endpoint like so: + +```ruby +get '/projects/:id/foo' do + Gitlab::QueryLimiting.disable!('...') + + # ... +end +``` diff --git a/doc/development/database/query_performance.md b/doc/development/database/query_performance.md new file mode 100644 index 00000000000..41dbd08d726 --- /dev/null +++ b/doc/development/database/query_performance.md @@ -0,0 +1,74 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Query performance guidelines + +This document describes various guidelines to follow when optimizing SQL queries. + +When you are optimizing your SQL queries, there are two dimensions to pay attention to: + +1. The query execution time. This is paramount as it reflects how the user experiences GitLab. +1. The query plan. Optimizing the query plan is important in allowing queries to independently scale over time. Realizing that an index keeps a query performing well as the table grows before the query degrades is an example of why we analyze these plans. + +## Timing guidelines for queries + +| Query Type | Maximum Query Time | Notes | +|----|----|---| +| General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. | +| Queries in a migration | `100ms` | This is different than the total [migration time](../migration_style_guide.md#how-long-a-migration-should-take). | +| Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | +| Background migrations | `1s` | | +| Service Ping | `1s` | See the [Service Ping docs](../service_ping/implement.md) for more details. | + +- When analyzing your query's performance, pay attention to if the time you are seeing is on a [cold or warm cache](#cold-and-warm-cache). These guidelines apply for both cache types. +- When working with batched queries, change the range and batch size to see how it effects the query timing and caching. +- If an existing query is not performing well, make an effort to improve it. If it is too complex or would stall development, create a follow-up so it can be addressed in a timely manner. You can always ask the database reviewer or maintainer for help and guidance. + +## Cold and warm cache + +When evaluating query performance it is important to understand the difference between +cold and warm cached queries. + +The first time a query is made, it is made on a "cold cache". Meaning it needs +to read from disk. If you run the query again, the data can be read from the +cache, or what PostgreSQL calls shared buffers. This is the "warm cache" query. + +When analyzing an [`EXPLAIN` plan](understanding_explain_plans.md), you can see +the difference not only in the timing, but by looking at the output for `Buffers` +by running your explain with `EXPLAIN(analyze, buffers)`. [Database Lab](understanding_explain_plans.md#database-lab-engine) +automatically includes these options. + +If you are making a warm cache query, you see only the `shared hits`. + +For example in #database-lab: + +```plaintext +Shared buffers: + - hits: 36467 (~284.90 MiB) from the buffer pool + - reads: 0 from the OS file cache, including disk I/O +``` + +Or in the explain plan from `psql`: + +```sql +Buffers: shared hit=7323 +``` + +If the cache is cold, you also see `reads`. + +In #database-lab: + +```plaintext +Shared buffers: + - hits: 17204 (~134.40 MiB) from the buffer pool + - reads: 15229 (~119.00 MiB) from the OS file cache, including disk I/O +``` + +In `psql`: + +```sql +Buffers: shared hit=7202 read=121 +``` diff --git a/doc/development/database/query_recorder.md b/doc/development/database/query_recorder.md new file mode 100644 index 00000000000..da5c6c8e6cb --- /dev/null +++ b/doc/development/database/query_recorder.md @@ -0,0 +1,145 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# QueryRecorder + +QueryRecorder is a tool for detecting the [N+1 queries problem](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations) from tests. + +> Implemented in [spec/support/query_recorder.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/helpers/query_recorder.rb) via [9c623e3e](https://gitlab.com/gitlab-org/gitlab-foss/commit/9c623e3e5d7434f2e30f7c389d13e5af4ede770a) + +As a rule, merge requests [should not increase query counts](../merge_request_performance_guidelines.md#query-counts). If you find yourself adding something like `.includes(:author, :assignee)` to avoid having `N+1` queries, consider using QueryRecorder to enforce this with a test. Without this, a new feature which causes an additional model to be accessed can silently reintroduce the problem. + +## How it works + +This style of test works by counting the number of SQL queries executed by ActiveRecord. First a control count is taken, then you add new records to the database and rerun the count. If the number of queries has significantly increased then an `N+1` queries problem exists. + +```ruby +it "avoids N+1 database queries" do + control = ActiveRecord::QueryRecorder.new { visit_some_page } + create_list(:issue, 5) + expect { visit_some_page }.not_to exceed_query_limit(control) +end +``` + +You can if you wish, have both the expectation and the control as +`QueryRecorder` instances: + +```ruby +it "avoids N+1 database queries" do + control = ActiveRecord::QueryRecorder.new { visit_some_page } + create_list(:issue, 5) + action = ActiveRecord::QueryRecorder.new { visit_some_page } + + expect(action).not_to exceed_query_limit(control) +end +``` + +As an example you might create 5 issues in between counts, which would cause the query count to increase by 5 if an N+1 problem exists. + +In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. + +If this test fails, and the control was passed as a `QueryRecorder`, then the +failure message indicates where the extra queries are by matching queries on +the longest common prefix, grouping similar queries together. + +## Cached queries + +By default, QueryRecorder ignores [cached queries](../merge_request_performance_guidelines.md#cached-queries) in the count. However, it may be better to count +all queries to avoid introducing an N+1 query that may be masked by the statement cache. +To do this, this requires the `:use_sql_query_cache` flag to be set. +You should pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher: + +```ruby +it "avoids N+1 database queries", :use_sql_query_cache do + control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } + create_list(:issue, 5) + expect { visit_some_page }.not_to exceed_all_query_limit(control) +end +``` + +## Use request specs instead of controller specs + +Use a [request spec](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/spec/requests) when writing a N+1 test on the controller level. + +Controller specs should not be used to write N+1 tests as the controller is only initialized once per example. +This could lead to false successes where subsequent "requests" could have queries reduced (for example, because of memoization). + +## Finding the source of the query + +There are multiple ways to find the source of queries. + +- Inspect the `QueryRecorder` `data` attribute. It stores queries by `file_name:line_number:method_name`. + Each entry is a `hash` with the following fields: + + - `count`: the number of times a query from this `file_name:line_number:method_name` was called + - `occurrences`: the actual `SQL` of each call + - `backtrace`: the stack trace of each call (if either of the two following options were enabled) + + `QueryRecorder#find_query` allows filtering queries by their `file_name:line_number:method_name` and + `count` attributes. For example: + + ```ruby + control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } + control.find_query(/.*note.rb.*/, 0, first_only: true) + ``` + + `QueryRecorder#occurrences_by_line_method` returns a sorted array based on `data`, sorted by `count`. + +- View the call backtrace for the specific `QueryRecorder` instance you want + by using `ActiveRecord::QueryRecorder.new(query_recorder_debug: true)`. The output + is stored in file `test.log`. + +- Enable the call backtrace for all tests using the `QUERY_RECORDER_DEBUG` environment variable. + + To enable this, run the specs with the `QUERY_RECORDER_DEBUG` environment variable set. For example: + + ```shell + QUERY_RECORDER_DEBUG=1 bundle exec rspec spec/requests/api/projects_spec.rb + ``` + + This logs calls to QueryRecorder into the `test.log` file. For example: + + ```sql + QueryRecorder SQL: SELECT COUNT(*) FROM "issues" WHERE "issues"."deleted_at" IS NULL AND "issues"."project_id" = $1 AND ("issues"."state" IN ('opened')) AND "issues"."confidential" = $2 + --> /home/user/gitlab/gdk/gitlab/spec/support/query_recorder.rb:19:in `callback' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:127:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `block in finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `each' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:36:in `finish' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:25:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:601:in `exec_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:585:in `execute_and_clear' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql/database_statements.rb:160:in `exec_query' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:356:in `select' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:32:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `block in select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:83:in `cache_sql' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `select_all' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:270:in `execute_simple_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:227:in `perform_calculation' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:133:in `calculate' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:48:in `count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:20:in `uncached_count' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `block in count' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `block in fetch' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:585:in `block in save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `block in instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications.rb:166:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `instrument' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:584:in `save_block_result_to_cache' + --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `fetch' + --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `count' + --> /home/user/gitlab/gdk/gitlab/app/models/project.rb:1296:in `open_issues_count' + ``` + +## See also + +- [Bullet](../profiling.md#bullet) For finding `N+1` query problems +- [Performance guidelines](../performance.md) +- [Merge request performance guidelines - Query counts](../merge_request_performance_guidelines.md#query-counts) +- [Merge request performance guidelines - Cached queries](../merge_request_performance_guidelines.md#cached-queries) diff --git a/doc/development/database/rename_database_tables.md b/doc/development/database/rename_database_tables.md index cbcbd507204..d6827cb9e03 100644 --- a/doc/development/database/rename_database_tables.md +++ b/doc/development/database/rename_database_tables.md @@ -81,10 +81,10 @@ Execute a standard migration (not a post-migration): when naming indexes, so there is a possibility that not all indexes are properly renamed. After running the migration locally, check if there are inconsistently named indexes (`db/structure.sql`). Those can be renamed manually in a separate migration, which can be also part of the release M.N+1. -- Foreign key columns might still contain the old table name. For smaller tables, follow our [standard column -rename process](avoiding_downtime_in_migrations.md#renaming-columns) +- Foreign key columns might still contain the old table name. For smaller tables, follow our + [standard column rename process](avoiding_downtime_in_migrations.md#renaming-columns) - Avoid renaming database tables which are using with triggers. -- Table modifications (add or remove columns) are not allowed during the rename process, please make sure that all changes to the table happen before the rename migration is started (or in the next release). +- Table modifications (add or remove columns) are not allowed during the rename process. Make sure that all changes to the table happen before the rename migration is started (or in the next release). - As the index names might change, verify that the model does not use bulk insert (for example, `insert_all` and `upsert_all`) with the `unique_by: index_name` option. Renaming an index while using these methods may break functionality. diff --git a/doc/development/database/serializing_data.md b/doc/development/database/serializing_data.md new file mode 100644 index 00000000000..97e6f665484 --- /dev/null +++ b/doc/development/database/serializing_data.md @@ -0,0 +1,90 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Serializing Data + +**Summary:** don't store serialized data in the database, use separate columns +and/or tables instead. This includes storing of comma separated values as a +string. + +Rails makes it possible to store serialized data in JSON, YAML or other formats. +Such a field can be defined as follows: + +```ruby +class Issue < ActiveRecord::Model + serialize :custom_fields +end +``` + +While it may be tempting to store serialized data in the database there are many +problems with this. This document outlines these problems and provide an +alternative. + +## Serialized Data Is Less Powerful + +When using a relational database you have the ability to query individual +fields, change the schema, index data, and so forth. When you use serialized data +all of that becomes either very difficult or downright impossible. While +PostgreSQL does offer the ability to query JSON fields it is mostly meant for +very specialized use cases, and not for more general use. If you use YAML in +turn there's no way to query the data at all. + +## Waste Of Space + +Storing serialized data such as JSON or YAML ends up wasting a lot of space. +This is because these formats often include additional characters (for example, double +quotes or newlines) besides the data that you are storing. + +## Difficult To Manage + +There comes a time where you must add a new field to the serialized +data, or change an existing one. Using serialized data this becomes difficult +and very time consuming as the only way of doing so is to re-write all the +stored values. To do so you would have to: + +1. Retrieve the data +1. Parse it into a Ruby structure +1. Mutate it +1. Serialize it back to a String +1. Store it in the database + +On the other hand, if one were to use regular columns adding a column would be: + +```sql +ALTER TABLE table_name ADD COLUMN column_name type; +``` + +Such a query would take very little to no time and would immediately apply to +all rows, without having to re-write large JSON or YAML structures. + +Finally, there comes a time when the JSON or YAML structure is no longer +sufficient and you must migrate away from it. When storing only a few rows +this may not be a problem, but when storing millions of rows such a migration +can take hours or even days to complete. + +## Relational Databases Are Not Document Stores + +When storing data as JSON or YAML you're essentially using your database as if +it were a document store (for example, MongoDB), except you're not using any of the +powerful features provided by a typical RDBMS _nor_ are you using any of the +features provided by a typical document store (for example, the ability to index fields +of documents with variable fields). In other words, it's a waste. + +## Consistent Fields + +One argument sometimes made in favour of serialized data is having to store +widely varying fields and values. Sometimes this is truly the case, and then +perhaps it might make sense to use serialized data. However, in 99% of the cases +the fields and types stored tend to be the same for every row. Even if there is +a slight difference you can still use separate columns and just not set the ones +you don't need. + +## The Solution + +The solution is to use separate columns and/or separate tables. +This allows you to use all the features provided by your database, it +makes it easier to manage and migrate the data, you conserve space, you can +index the data efficiently and so forth. diff --git a/doc/development/database/sha1_as_binary.md b/doc/development/database/sha1_as_binary.md new file mode 100644 index 00000000000..dab9b0fe72e --- /dev/null +++ b/doc/development/database/sha1_as_binary.md @@ -0,0 +1,42 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Storing SHA1 Hashes As Binary + +Storing SHA1 hashes as strings is not very space efficient. A SHA1 as a string +requires at least 40 bytes, an additional byte to store the encoding, and +perhaps more space depending on the internals of PostgreSQL. + +On the other hand, if one were to store a SHA1 as binary one would only need 20 +bytes for the actual SHA1, and 1 or 4 bytes of additional space (again depending +on database internals). This means that in the best case scenario we can reduce +the space usage by 50%. + +To make this easier to work with you can include the concern `ShaAttribute` into +a model and define a SHA attribute using the `sha_attribute` class method. For +example: + +```ruby +class Commit < ActiveRecord::Base + include ShaAttribute + + sha_attribute :sha +end +``` + +This allows you to use the value of the `sha` attribute as if it were a string, +while storing it as binary. This means that you can do something like this, +without having to worry about converting data to the right binary format: + +```ruby +commit = Commit.find_by(sha: '88c60307bd1f215095834f09a1a5cb18701ac8ad') +commit.sha = '971604de4cfa324d91c41650fabc129420c8d1cc' +commit.save +``` + +There is however one requirement: the column used to store the SHA has _must_ be +a binary type. For Rails this means you need to use the `:binary` type instead +of `:text` or `:string`. diff --git a/doc/development/database/single_table_inheritance.md b/doc/development/database/single_table_inheritance.md new file mode 100644 index 00000000000..c8d082e8a67 --- /dev/null +++ b/doc/development/database/single_table_inheritance.md @@ -0,0 +1,63 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Single Table Inheritance + +**Summary:** don't use Single Table Inheritance (STI), use separate tables +instead. + +Rails makes it possible to have multiple models stored in the same table and map +these rows to the correct models using a `type` column. This can be used to for +example store two different types of SSH keys in the same table. + +While tempting to use one should avoid this at all costs for the same reasons as +outlined in the document ["Polymorphic Associations"](polymorphic_associations.md). + +## Solution + +The solution is very simple: just use a separate table for every type you'd +otherwise store in the same table. For example, instead of having a `keys` table +with `type` set to either `Key` or `DeployKey` you'd have two separate tables: +`keys` and `deploy_keys`. + +## In migrations + +Whenever a model is used in a migration, single table inheritance should be disabled. +Due to the way Rails loads associations (even in migrations), failing to disable STI +could result in loading unexpected code or associations which may cause unintended +side effects or failures during upgrades. + +```ruby +class SomeMigration < Gitlab::Database::Migration[2.0] + class Services < MigrationRecord + self.table_name = 'services' + self.inheritance_column = :_type_disabled + end + + def up + ... +``` + +If nothing needs to be added to the model other than disabling STI or `EachBatch`, +use the helper `define_batchable_model` instead of defining the class. +This ensures that the migration loads the columns for the migration in isolation, +and the helper disables STI by default. + +```ruby +class EnqueueSomeBackgroundMigration < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + define_batchable_model('services').select(:id).in_batches do |relation| + jobs = relation.pluck(:id).map do |id| + ['ExtractServicesUrl', [id]] + end + + BackgroundMigrationWorker.bulk_perform_async(jobs) + end + end + ... +``` diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index 73e023f8d45..e2e1191018b 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -148,8 +148,9 @@ to update the `title_html` with a title that has more than 1024 characters, the a database error. Adding or removing a constraint to an existing attribute requires that any application changes are -deployed _first_, [otherwise servers still in the old version of the application may try to update the -attribute with invalid values](../multi_version_compatibility.md#ci-artifact-uploads-were-failing). +deployed _first_, +otherwise servers still in the old version of the application +[may try to update the attribute with invalid values](../multi_version_compatibility.md#ci-artifact-uploads-were-failing). For these reasons, `add_text_limit` should run in a post-deployment migration. Still in our example, for the 13.0 milestone (current), consider that the following validation @@ -188,7 +189,7 @@ migration or a background data migration: - If the data volume is less than `1,000` records, then the data migration can be executed within the post-migration. - If the data volume is higher than `1,000` records, it's advised to create a background migration. -When unsure about which option to use, please contact the Database team for advice. +When unsure about which option to use, contact the Database team for advice. Back to our example, the issues table is considerably large and frequently accessed, so we are going to add a background migration for the 13.0 milestone (current), diff --git a/doc/development/database/swapping_tables.md b/doc/development/database/swapping_tables.md new file mode 100644 index 00000000000..efb481ccf35 --- /dev/null +++ b/doc/development/database/swapping_tables.md @@ -0,0 +1,51 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Swapping Tables + +Sometimes you need to replace one table with another. For example, when +migrating data in a very large table it's often better to create a copy of the +table and insert & migrate the data into this new table in the background. + +Let's say you want to swap the table `events` with `events_for_migration`. In +this case you need to follow 3 steps: + +1. Rename `events` to `events_temporary` +1. Rename `events_for_migration` to `events` +1. Rename `events_temporary` to `events_for_migration` + +Rails allows you to do this using the `rename_table` method: + +```ruby +rename_table :events, :events_temporary +rename_table :events_for_migration, :events +rename_table :events_temporary, :events_for_migration +``` + +This does not require any downtime as long as the 3 `rename_table` calls are +executed in the _same_ database transaction. Rails by default uses database +transactions for migrations, but if it doesn't you need to start one +manually: + +```ruby +Event.transaction do + rename_table :events, :events_temporary + rename_table :events_for_migration, :events + rename_table :events_temporary, :events_for_migration +end +``` + +Once swapped you _have to_ reset the primary key of the new table. For +PostgreSQL you can use the `reset_pk_sequence!` method like so: + +```ruby +reset_pk_sequence!('events') +``` + +Failure to reset the primary keys results in newly created rows starting +with an ID value of 1. Depending on the existing data this can then lead to +duplicate key constraints from popping up, preventing users from creating new +data. diff --git a/doc/development/database/transaction_guidelines.md b/doc/development/database/transaction_guidelines.md index 255de19a420..1583bbc02c2 100644 --- a/doc/development/database/transaction_guidelines.md +++ b/doc/development/database/transaction_guidelines.md @@ -12,7 +12,7 @@ For further reference, check PostgreSQL documentation about [transactions](https ## Database decomposition and sharding -The [sharding group](https://about.gitlab.com/handbook/engineering/development/enablement/sharding/) plans +The [Pods Group](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/pods/) plans to split the main GitLab database and move some of the database tables to other database servers. We start decomposing the `ci_*`-related database tables first. To maintain the current application diff --git a/doc/development/database/understanding_explain_plans.md b/doc/development/database/understanding_explain_plans.md new file mode 100644 index 00000000000..446a84d5232 --- /dev/null +++ b/doc/development/database/understanding_explain_plans.md @@ -0,0 +1,829 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Understanding EXPLAIN plans + +PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This +command can be invaluable when trying to determine how a query performs. +You can use this command directly in your SQL query, as long as the query starts +with it: + +```sql +EXPLAIN +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +When running this on GitLab.com, we are presented with the following output: + +```sql +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +When using _just_ `EXPLAIN`, PostgreSQL does not actually execute our query, +instead it produces an _estimated_ execution plan based on the available +statistics. This means the actual plan can differ quite a bit. Fortunately, +PostgreSQL provides us with the option to execute the query as well. To do so, +we need to use `EXPLAIN ANALYZE` instead of just `EXPLAIN`: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This produces: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +As we can see this plan is quite different, and includes a lot more data. Let's +discuss this step by step. + +Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a +query that writes data or might time out. If the query modifies data, +consider wrapping it in a transaction that rolls back automatically like so: + +```sql +BEGIN; +EXPLAIN ANALYZE +DELETE FROM users WHERE id = 1; +ROLLBACK; +``` + +The `EXPLAIN` command also takes additional options, such as `BUFFERS`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +This then produces: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +For more information, refer to the official +[`EXPLAIN` documentation](https://www.postgresql.org/docs/current/sql-explain.html) +and [using `EXPLAIN` guide](https://www.postgresql.org/docs/current/using-explain.html). + +## Nodes + +Every query plan consists of nodes. Nodes can be nested, and are executed from +the inside out. This means that the innermost node is executed before an outer +node. This can be best thought of as nested function calls, returning their +results as they unwind. For example, a plan starting with an `Aggregate` +followed by a `Nested Loop`, followed by an `Index Only scan` can be thought of +as the following Ruby code: + +```ruby +aggregate( + nested_loop( + index_only_scan() + index_only_scan() + ) +) +``` + +Nodes are indicated using a `->` followed by the type of node taken. For +example: + +```sql +Aggregate (cost=922411.76..922411.77 rows=1 width=8) + -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) +``` + +Here the first node executed is `Seq scan on projects`. The `Filter:` is an +additional filter applied to the results of the node. A filter is very similar +to Ruby's `Array#select`: it takes the input rows, applies the filter, and +produces a new list of rows. After the node is done, we perform the `Aggregate` +above it. + +Nested nodes look like this: + +```sql +Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) + Buffers: shared hit=155 + -> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +Planning time: 2.585 ms +Execution time: 0.310 ms +``` + +Here we first perform two separate "Index Only" scans, followed by performing a +"Nested Loop" on the result of these two scans. + +## Node statistics + +Each node in a plan has a set of associated statistics, such as the cost, the +number of rows produced, the number of loops performed, and more. For example: + +```sql +Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) +``` + +Here we can see that our cost ranges from `0.00..908044.47` (we cover this in +a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN +ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` +statistics describes the estimated width of each row, in bytes. + +The `costs` field specifies how expensive a node was. The cost is measured in +arbitrary units determined by the query planner's cost parameters. What +influences the costs depends on a variety of settings, such as `seq_page_cost`, +`cpu_tuple_cost`, and various others. +The format of the costs field is as follows: + +```sql +STARTUP COST..TOTAL COST +``` + +The startup cost states how expensive it was to start the node, with the total +cost describing how expensive the entire node was. In general: the greater the +values, the more expensive the node. + +When using `EXPLAIN ANALYZE`, these statistics also include the actual time +(in milliseconds) spent, and other runtime statistics (for example, the actual number of +produced rows): + +```sql +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) +``` + +Here we can see we estimated 5,746,969 rows to be returned, but in reality we +returned 5,746,940 rows. We can also see that _just_ this sequential scan took +2.98 seconds to run. + +Using `EXPLAIN (ANALYZE, BUFFERS)` also gives us information about the +number of rows removed by a filter, the number of buffers used, and more. For +example: + +```sql +Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +``` + +Here we can see that our filter has to remove 65,677 rows, and that we use +208,846 buffers. Each buffer in PostgreSQL is 8 KB (8192 bytes), meaning our +above node uses *1.6 GB of buffers*. That's a lot! + +Keep in mind that some statistics are per-loop averages, while others are total values: + +| Field name | Value type | +| --- | --- | +| Actual Total Time | per-loop average | +| Actual Rows | per-loop average | +| Buffers Shared Hit | total value | +| Buffers Shared Read | total value | +| Buffers Shared Dirtied | total value | +| Buffers Shared Written | total value | +| I/O Read Time | total value | +| I/O Read Write | total value | + +For example: + +```sql + -> Index Scan using users_pkey on public.users (cost=0.43..3.44 rows=1 width=1318) (actual time=0.025..0.025 rows=1 loops=888) + Index Cond: (users.id = issues.author_id) + Buffers: shared hit=3543 read=9 + I/O Timings: read=17.760 write=0.000 +``` + +Here we can see that this node used 3552 buffers (3543 + 9), returned 888 rows (`888 * 1`), and the actual duration was 22.2 milliseconds (`888 * 0.025`). +17.76 milliseconds of the total duration was spent in reading from disk, to retrieve data that was not in the cache. + +## Node types + +There are quite a few different types of nodes, so we only cover some of the +more common ones here. + +A full list of all the available nodes and their descriptions can be found in +the [PostgreSQL source file `plannodes.h`](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h). +pgMustard's [EXPLAIN docs](https://www.pgmustard.com/docs/explain) also offer detailed look into nodes and their fields. + +### Seq Scan + +A sequential scan over (a chunk of) a database table. This is like using +`Array#each`, but on a database table. Sequential scans can be quite slow when +retrieving lots of rows, so it's best to avoid these for large tables. + +### Index Only Scan + +A scan on an index that did not require fetching anything from the table. In +certain cases an index only scan may still fetch data from the table, in this +case the node includes a `Heap Fetches:` statistic. + +### Index Scan + +A scan on an index that required retrieving some data from the table. + +### Bitmap Index Scan and Bitmap Heap scan + +Bitmap scans fall between sequential scans and index scans. These are typically +used when we would read too much data from an index scan, but too little to +perform a sequential scan. A bitmap scan uses what is known as a +[bitmap index](https://en.wikipedia.org/wiki/Bitmap_index) to perform its work. + +The [source code of PostgreSQL](https://gitlab.com/postgres/postgres/blob/REL_11_STABLE/src/include/nodes/plannodes.h#L441) +states the following on bitmap scans: + +> Bitmap Index Scan delivers a bitmap of potential tuple locations; it does not +> access the heap itself. The bitmap is used by an ancestor Bitmap Heap Scan +> node, possibly after passing through intermediate Bitmap And and/or Bitmap Or +> nodes to combine it with the results of other Bitmap Index Scans. + +### Limit + +Applies a `LIMIT` on the input rows. + +### Sort + +Sorts the input rows as specified using an `ORDER BY` statement. + +### Nested Loop + +A nested loop executes its child nodes for every row produced by a node that +precedes it. For example: + +```sql +-> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) + Buffers: shared hit=155 + -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) + Index Cond: (id < 100) + Heap Fetches: 0 + -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) + Index Cond: (id = users_1.id) + Heap Fetches: 0 +``` + +Here the first child node (`Index Only Scan using users_pkey on users users_1`) +produces 36 rows, and is executed once (`rows=36 loops=1`). The next node +produces 1 row (`rows=1`), but is repeated 36 times (`loops=36`). This is +because the previous node produced 36 rows. + +This means that nested loops can quickly slow the query down if the various +child nodes keep producing many rows. + +## Optimising queries + +With that out of the way, let's see how we can optimise a query. Let's use the +following query as an example: + +```sql +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This query counts the number of users that have a Twitter profile set. +Let's run this using `EXPLAIN (ANALYZE, BUFFERS)`: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM users +WHERE twitter != ''; +``` + +This produces the following plan: + +```sql +Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) + Buffers: shared hit=202662 + -> Seq Scan on users (cost=0.00..844969.99 rows=56087 width=0) (actual time=0.019..1265.883 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487813 + Buffers: shared hit=202662 +Planning time: 0.390 ms +Execution time: 1271.180 ms +``` + +From this query plan we can see the following: + +1. We need to perform a sequential scan on the `users` table. +1. This sequential scan filters out 2,487,813 rows using a `Filter`. +1. We use 202,622 buffers, which equals 1.58 GB of memory. +1. It takes us 1.2 seconds to do all of this. + +Considering we are just counting users, that's quite expensive! + +Before we start making any changes, let's see if there are any existing indexes +on the `users` table that we might be able to use. We can obtain this +information by running `\d users` in a `psql` console, then scrolling down to +the `Indexes:` section: + +```sql +Indexes: + "users_pkey" PRIMARY KEY, btree (id) + "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token) + "index_users_on_email" UNIQUE, btree (email) + "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token) + "index_users_on_static_object_token" UNIQUE, btree (static_object_token) + "index_users_on_unlock_token" UNIQUE, btree (unlock_token) + "index_on_users_name_lower" btree (lower(name::text)) + "index_users_on_accepted_term_id" btree (accepted_term_id) + "index_users_on_admin" btree (admin) + "index_users_on_created_at" btree (created_at) + "index_users_on_email_trigram" gin (email gin_trgm_ops) + "index_users_on_feed_token" btree (feed_token) + "index_users_on_group_view" btree (group_view) + "index_users_on_incoming_email_token" btree (incoming_email_token) + "index_users_on_managing_group_id" btree (managing_group_id) + "index_users_on_name" btree (name) + "index_users_on_name_trigram" gin (name gin_trgm_ops) + "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text + "index_users_on_state" btree (state) + "index_users_on_state_and_user_type" btree (state, user_type) + "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL + "index_users_on_user_type" btree (user_type) + "index_users_on_username" btree (username) + "index_users_on_username_trigram" gin (username gin_trgm_ops) + "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text +``` + +Here we can see there is no index on the `twitter` column, which means +PostgreSQL has to perform a sequential scan in this case. Let's try to fix this +by adding the following index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +If we now re-run our query using `EXPLAIN (ANALYZE, BUFFERS)` we get the +following plan: + +```sql +Aggregate (cost=61002.82..61002.83 rows=1 width=8) (actual time=297.311..297.312 rows=1 loops=1) + Buffers: shared hit=51854 dirtied=19 + -> Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 + Heap Fetches: 26037 + Buffers: shared hit=51854 dirtied=19 +Planning time: 0.191 ms +Execution time: 297.334 ms +``` + +Now it takes just under 300 milliseconds to get our data, instead of 1.2 +seconds. However, we still use 51,854 buffers, which is about 400 MB of memory. +300 milliseconds is also quite slow for such a simple query. To understand why +this query is still expensive, let's take a look at the following: + +```sql +Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) + Filter: ((twitter)::text <> ''::text) + Rows Removed by Filter: 2487830 +``` + +We start with an index only scan on our index, but we somehow still apply a +`Filter` that filters out 2,487,830 rows. Why is that? Well, let's look at how +we created the index: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); +``` + +We told PostgreSQL to index all possible values of the `twitter` column, +even empty strings. Our query in turn uses `WHERE twitter != ''`. This means +that the index does improve things, as we don't need to do a sequential scan, +but we may still encounter empty strings. This means PostgreSQL _has_ to apply a +Filter on the index results to get rid of those values. + +Fortunately, we can improve this even further using "partial indexes". Partial +indexes are indexes with a `WHERE` condition that is applied when indexing data. +For example: + +```sql +CREATE INDEX CONCURRENTLY some_index ON users (email) WHERE id < 100 +``` + +This index would only index the `email` value of rows that match `WHERE id < +100`. We can use partial indexes to change our Twitter index to the following: + +```sql +CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; +``` + +After being created, if we run our query again we are given the following plan: + +```sql +Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) + Buffers: shared hit=44036 + -> Index Only Scan using twitter_test on users (cost=0.41..1479.71 rows=51420 width=0) (actual time=0.023..15.514 rows=51833 loops=1) + Heap Fetches: 1208 + Buffers: shared hit=44036 +Planning time: 0.123 ms +Execution time: 19.848 ms +``` + +That's _a lot_ better! Now it only takes 20 milliseconds to get the data, and we +only use about 344 MB of buffers (instead of the original 1.58 GB). The reason +this works is that now PostgreSQL no longer needs to apply a `Filter`, as the +index only contains `twitter` values that are not empty. + +Keep in mind that you shouldn't just add partial indexes every time you want to +optimise a query. Every index has to be updated for every write, and they may +require quite a bit of space, depending on the amount of indexed data. As a +result, first check if there are any existing indexes you may be able to reuse. +If there aren't any, check if you can perhaps slightly change an existing one to +fit both the existing and new queries. Only add a new index if none of the +existing indexes can be used in any way. + +When comparing execution plans, don't take timing as the only important metric. +Good timing is the main goal of any optimization, but it can be too volatile to +be used for comparison (for example, it depends a lot on the state of cache). +When optimizing a query, we usually need to reduce the amount of data we're +dealing with. Indexes are the way to work with fewer pages (buffers) to get the +result, so, during optimization, look at the number of buffers used (read and hit), +and work on reducing these numbers. Reduced timing is the consequence of reduced +buffer numbers. [Database Lab Engine](#database-lab-engine) guarantees that the plan is structurally +identical to production (and overall number of buffers is the same as on production), +but difference in cache state and I/O speed may lead to different timings. + +## Queries that can't be optimised + +Now that we have seen how to optimise a query, let's look at another query that +we might not be able to optimise: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT COUNT(*) +FROM projects +WHERE visibility_level IN (0, 20); +``` + +The output of `EXPLAIN (ANALYZE, BUFFERS)` is as follows: + +```sql +Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) + Buffers: shared hit=208846 + -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 65677 + Buffers: shared hit=208846 +Planning time: 2.861 ms +Execution time: 3428.596 ms +``` + +Looking at the output we see the following Filter: + +```sql +Filter: (visibility_level = ANY ('{0,20}'::integer[])) +Rows Removed by Filter: 65677 +``` + +Looking at the number of rows removed by the filter, we may be tempted to add an +index on `projects.visibility_level` to somehow turn this Sequential scan + +filter into an index-only scan. + +Unfortunately, doing so is unlikely to improve anything. Contrary to what some +might believe, an index being present _does not guarantee_ that PostgreSQL +actually uses it. For example, when doing a `SELECT * FROM projects` it is much +cheaper to just scan the entire table, instead of using an index and then +fetching data from the table. In such cases PostgreSQL may decide to not use an +index. + +Second, let's think for a moment what our query does: it gets all projects with +visibility level 0 or 20. In the above plan we can see this produces quite a lot +of rows (5,745,940), but how much is that relative to the total? Let's find out +by running the following query: + +```sql +SELECT visibility_level, count(*) AS amount +FROM projects +GROUP BY visibility_level +ORDER BY visibility_level ASC; +``` + +For GitLab.com this produces: + +```sql + visibility_level | amount +------------------+--------- + 0 | 5071325 + 10 | 65678 + 20 | 674801 +``` + +Here the total number of projects is 5,811,804, and 5,746,126 of those are of +level 0 or 20. That's 98% of the entire table! + +So no matter what we do, this query retrieves 98% of the entire table. Since +most time is spent doing exactly that, there isn't really much we can do to +improve this query, other than _not_ running it at all. + +What is important here is that while some may recommend to straight up add an +index the moment you see a sequential scan, it is _much more important_ to first +understand what your query does, how much data it retrieves, and so on. After +all, you can not optimise something you do not understand. + +### Cardinality and selectivity + +Earlier we saw that our query had to retrieve 98% of the rows in the table. +There are two terms commonly used for databases: cardinality, and selectivity. +Cardinality refers to the number of unique values in a particular column in a +table. + +Selectivity is the number of unique values produced by an operation (for example, an +index scan or filter), relative to the total number of rows. The higher the +selectivity, the more likely PostgreSQL is able to use an index. + +In the above example, there are only 3 unique values: 0, 10, and 20. This means +the cardinality is 3. The selectivity in turn is also very low: 0.0000003% (2 / +5,811,804), because our `Filter` only filters using two values (`0` and `20`). +With such a low selectivity value it's not surprising that PostgreSQL decides +using an index is not worth it, because it would produce almost no unique rows. + +## Rewriting queries + +So the above query can't really be optimised as-is, or at least not much. But +what if we slightly change the purpose of it? What if instead of retrieving all +projects with `visibility_level` 0 or 20, we retrieve those that a user +interacted with somehow? + +Fortunately, GitLab has an answer for this, and it's a table called +`user_interacted_projects`. This table has the following schema: + +```sql +Table "public.user_interacted_projects" + Column | Type | Modifiers +------------+---------+----------- + user_id | integer | not null + project_id | integer | not null +Indexes: + "index_user_interacted_projects_on_project_id_and_user_id" UNIQUE, btree (project_id, user_id) + "index_user_interacted_projects_on_user_id" btree (user_id) +Foreign-key constraints: + "fk_rails_0894651f08" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +``` + +Let's rewrite our query to `JOIN` this table onto our projects, and get the +projects for a specific user: + +```sql +EXPLAIN ANALYZE +SELECT COUNT(*) +FROM projects +INNER JOIN user_interacted_projects ON user_interacted_projects.project_id = projects.id +WHERE projects.visibility_level IN (0, 20) +AND user_interacted_projects.user_id = 1; +``` + +What we do here is the following: + +1. Get our projects. +1. `INNER JOIN` `user_interacted_projects`, meaning we're only left with rows in + `projects` that have a corresponding row in `user_interacted_projects`. +1. Limit this to the projects with `visibility_level` of 0 or 20, and to + projects that the user with ID 1 interacted with. + +If we run this query we get the following plan: + +```sql + Aggregate (cost=871.03..871.04 rows=1 width=8) (actual time=9.763..9.763 rows=1 loops=1) + -> Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) + -> Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) + Index Cond: (user_id = 1) + -> Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 + Planning time: 2.614 ms + Execution time: 9.809 ms +``` + +Here it only took us just under 10 milliseconds to get the data. We can also see +we're retrieving far fewer projects: + +```sql +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) + Index Cond: (id = user_interacted_projects.project_id) + Filter: (visibility_level = ANY ('{0,20}'::integer[])) + Rows Removed by Filter: 0 +``` + +Here we see we perform 145 loops (`loops=145`), with every loop producing 1 row +(`rows=1`). This is much less than before, and our query performs much better! + +If we look at the plan we also see our costs are very low: + +```sql +Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) +``` + +Here our cost is only 3.45, and it takes us 7.25 milliseconds to do so (0.05 * 145). +The next index scan is a bit more expensive: + +```sql +Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) +``` + +Here the cost is 160.71 (`cost=0.43..160.71`), taking about 2.5 milliseconds +(based on the output of `actual time=....`). + +The most expensive part here is the "Nested Loop" that acts upon the result of +these two index scans: + +```sql +Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) +``` + +Here we had to perform 870.52 disk page fetches for 203 rows, 9.748 +milliseconds, producing 143 rows in a single loop. + +The key takeaway here is that sometimes you have to rewrite (parts of) a query +to make it better. Sometimes that means having to slightly change your feature +to accommodate for better performance. + +## What makes a bad plan + +This is a bit of a difficult question to answer, because the definition of "bad" +is relative to the problem you are trying to solve. However, some patterns are +best avoided in most cases, such as: + +- Sequential scans on large tables +- Filters that remove a lot of rows +- Performing a certain step that requires _a lot_ of + buffers (for example, an index scan for GitLab.com that requires more than 512 MB). + +As a general guideline, aim for a query that: + +1. Takes no more than 10 milliseconds. Our target time spent in SQL per request + is around 100 milliseconds, so every query should be as fast as possible. +1. Does not use an excessive number of buffers, relative to the workload. For + example, retrieving ten rows shouldn't require 1 GB of buffers. +1. Does not spend a long amount of time performing disk IO operations. The + setting `track_io_timing` must be enabled for this data to be included in the + output of `EXPLAIN ANALYZE`. +1. Applies a `LIMIT` when retrieving rows without aggregating them, such as + `SELECT * FROM users`. +1. Doesn't use a `Filter` to filter out too many rows, especially if the query + does not use a `LIMIT` to limit the number of returned rows. Filters can + usually be removed by adding a (partial) index. + +These are _guidelines_ and not hard requirements, as different needs may require +different queries. The only _rule_ is that you _must always measure_ your query +(preferably using a production-like database) using `EXPLAIN (ANALYZE, BUFFERS)` +and related tools such as: + +- [`explain.depesz.com`](https://explain.depesz.com/). +- [`explain.dalibo.com/`](https://explain.dalibo.com/). + +## Producing query plans + +There are a few ways to get the output of a query plan. Of course you +can directly run the `EXPLAIN` query in the `psql` console, or you can +follow one of the other options below. + +### Database Lab Engine + +GitLab team members can use [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab), and the companion +SQL optimization tool - [Joe Bot](https://gitlab.com/postgres-ai/joe). + +Database Lab Engine provides developers with their own clone of the production database, while Joe Bot helps with exploring execution plans. + +Joe Bot is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack, +and through its [web interface](https://console.postgres.ai/gitlab/joe-instances). + +With Joe Bot you can execute DDL statements (like creating indexes, tables, and columns) and get query plans for `SELECT`, `UPDATE`, and `DELETE` statements. + +For example, in order to test new index on a column that is not existing on production yet, you can do the following: + +Create the column: + +```sql +exec ALTER TABLE projects ADD COLUMN last_at timestamp without time zone +``` + +Create the index: + +```sql +exec CREATE INDEX index_projects_last_activity ON projects (last_activity_at) WHERE last_activity_at IS NOT NULL +``` + +Analyze the table to update its statistics: + +```sql +exec ANALYZE projects +``` + +Get the query plan: + +```sql +explain SELECT * FROM projects WHERE last_activity_at < CURRENT_DATE +``` + +Once done you can rollback your changes: + +```sql +reset +``` + +For more information about the available options, run: + +```sql +help +``` + +The web interface comes with the following execution plan visualizers included: + +- [Depesz](https://explain.depesz.com/) +- [PEV2](https://github.com/dalibo/pev2) +- [FlameGraph](https://github.com/mgartner/pg_flame) + +#### Tips & Tricks + +The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings are applied to all subsequent commands until you reset them. For example you can disable parallel queries with + +```sql +exec SET max_parallel_workers_per_gather = 0 +``` + +### Rails console + +Using the [`activerecord-explain-analyze`](https://github.com/6/activerecord-explain-analyze) +you can directly generate the query plan from the Rails console: + +```ruby +pry(main)> require 'activerecord-explain-analyze' +=> true +pry(main)> Project.where('build_timeout > ?', 3600).explain(analyze: true) + Project Load (1.9ms) SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) + ↳ (pry):12 +=> EXPLAIN for: SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) +Seq Scan on public.projects (cost=0.00..2.17 rows=1 width=742) (actual time=0.040..0.041 rows=0 loops=1) + Output: id, name, path, description, created_at, updated_at, creator_id, namespace_id, ... + Filter: (projects.build_timeout > 3600) + Rows Removed by Filter: 14 + Buffers: shared hit=2 +Planning time: 0.411 ms +Execution time: 0.113 ms +``` + +### ChatOps + +GitLab team members can also use our ChatOps solution, available in Slack +using the [`/chatops` slash command](../chatops_on_gitlabcom.md). + +NOTE: +While ChatOps is still available, the recommended way to generate execution plans is to use [Database Lab Engine](#database-lab-engine). + +You can use ChatOps to get a query plan by running the following: + +```sql +/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Visualising the plan using <https://explain.depesz.com/> is also supported: + +```sql +/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) +``` + +Quoting the query is not necessary. + +For more information about the available options, run: + +```sql +/chatops run explain --help +``` + +## Further reading + +A more extensive guide on understanding query plans can be found in +the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) +from [Dalibo.org](https://www.dalibo.com/en/). + +Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. diff --git a/doc/development/database/verifying_database_capabilities.md b/doc/development/database/verifying_database_capabilities.md new file mode 100644 index 00000000000..55347edf4ec --- /dev/null +++ b/doc/development/database/verifying_database_capabilities.md @@ -0,0 +1,38 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Verifying Database Capabilities + +Sometimes certain bits of code may only work on a certain database +version. While we try to avoid such code as much as possible sometimes it is +necessary to add database (version) specific behavior. + +To facilitate this we have the following methods that you can use: + +- `ApplicationRecord.database.version`: returns the PostgreSQL version number as a string + in the format `X.Y.Z`. + +This allows you to write code such as: + +```ruby +if ApplicationRecord.database.version.to_f >= 11.7 + run_really_fast_query +else + run_fast_query +end +``` + +## Read-only database + +The database can be used in read-only mode. In this case we have to +make sure all GET requests don't attempt any write operations to the +database. If one of those requests wants to write to the database, it needs +to be wrapped in a `Gitlab::Database.read_only?` or `Gitlab::Database.read_write?` +guard, to make sure it doesn't for read-only databases. + +We have a Rails Middleware that filters any potentially writing +operations (the `CUD` operations of CRUD) and prevent the user from trying +to update the database and getting a 500 error (see `Gitlab::Middleware::ReadOnly`). diff --git a/doc/development/database_debugging.md b/doc/development/database_debugging.md index 5d46ade98bb..f18830ee7ca 100644 --- a/doc/development/database_debugging.md +++ b/doc/development/database_debugging.md @@ -1,177 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/database_debugging.md' +remove_date: '2022-11-06' --- -# Troubleshooting and Debugging Database +This document was moved to [another location](database/database_debugging.md). -This section is to help give some copy-pasta you can use as a reference when you -run into some head-banging database problems. - -A first step is to search for your error in Slack, or search for `GitLab <my error>` with Google. - -Available `RAILS_ENV`: - -- `production` (generally not for your main GDK database, but you may need this for other installations such as Omnibus). -- `development` (this is your main GDK db). -- `test` (used for tests like RSpec). - -## Delete everything and start over - -If you just want to delete everything and start over with an empty DB (approximately 1 minute): - -```shell -bundle exec rake db:reset RAILS_ENV=development -``` - -If you want to seed the empty DB with sample data (approximately 4 minutes): - -```shell -bundle exec rake dev:setup -``` - -If you just want to delete everything and start over with sample data (approximately 4 minutes). This -also does `db:reset` and runs DB-specific migrations: - -```shell -bundle exec rake db:setup RAILS_ENV=development -``` - -If your test DB is giving you problems, it is safe to delete everything because it doesn't contain important -data: - -```shell -bundle exec rake db:reset RAILS_ENV=test -``` - -## Migration wrangling - -- `bundle exec rake db:migrate RAILS_ENV=development`: Execute any pending migrations that you may have picked up from a MR -- `bundle exec rake db:migrate:status RAILS_ENV=development`: Check if all migrations are `up` or `down` -- `bundle exec rake db:migrate:down VERSION=20170926203418 RAILS_ENV=development`: Tear down a migration -- `bundle exec rake db:migrate:up VERSION=20170926203418 RAILS_ENV=development`: Set up a migration -- `bundle exec rake db:migrate:redo VERSION=20170926203418 RAILS_ENV=development`: Re-run a specific migration - -## Manually access the database - -Access the database via one of these commands (they all get you to the same place) - -```shell -gdk psql -d gitlabhq_development -bundle exec rails dbconsole -e development -bundle exec rails db -e development -``` - -- `\q`: Quit/exit -- `\dt`: List all tables -- `\d+ issues`: List columns for `issues` table -- `CREATE TABLE board_labels();`: Create a table called `board_labels` -- `SELECT * FROM schema_migrations WHERE version = '20170926203418';`: Check if a migration was run -- `DELETE FROM schema_migrations WHERE version = '20170926203418';`: Manually remove a migration - -## Access the database with a GUI - -Most GUIs (DataGrip, RubyMine, DBeaver) require a TCP connection to the database, but by default -the database runs on a UNIX socket. To be able to access the database from these tools, some steps -are needed: - -1. On the GDK root directory, run: - - ```shell - gdk config set postgresql.host localhost - ``` - -1. Open your `gdk.yml`, and confirm that it has the following lines: - - ```yaml - postgresql: - host: localhost - ``` - -1. Reconfigure GDK: - - ```shell - gdk reconfigure - ``` - -1. On your database GUI, select `localhost` as host, `5432` as port and `gitlabhq_development` as database. - Alternatively, you can use the connection string `postgresql://localhost:5432/gitlabhq_development`. - -The new connection should be working now. - -## Access the GDK database with Visual Studio Code - -Use these instructions for exploring the GitLab database while developing with the GDK: - -1. Install or open [Visual Studio Code](https://code.visualstudio.com/download). -1. Install the [PostgreSQL VSCode Extension](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres). -1. In Visual Studio Code select **PostgreSQL Explorer** in the left toolbar. -1. In the top bar of the new window, select `+` to **Add Database Connection**, and follow the prompts to fill in the details: - 1. **Hostname**: the path to the PostgreSQL folder in your GDK directory (for example `/dev/gitlab-development-kit/postgresql`). - 1. **PostgreSQL user to authenticate as**: usually your local username, unless otherwise specified during PostgreSQL installation. - 1. **Password of the PostgreSQL user**: the password you set when installing PostgreSQL. - 1. **Port number to connect to**: `5432` (default). - 1. **Use an SSL connection?** This depends on your installation. Options are: - - **Use Secure Connection** - - **Standard Connection** (default) - 1. **Optional. The database to connect to**: `gitlabhq_development`. - 1. **The display name for the database connection**: `gitlabhq_development`. - -Your database connection should now be displayed in the PostgreSQL Explorer pane and -you can explore the `gitlabhq_development` database. If you cannot connect, ensure -that GDK is running. For further instructions on how to use the PostgreSQL Explorer -Extension for Visual Studio Code, read the [usage section](https://marketplace.visualstudio.com/items?itemName=ckolkman.vscode-postgres#usage) -of the extension documentation. - -## FAQ - -### `ActiveRecord::PendingMigrationError` with Spring - -When running specs with the [Spring pre-loader](rake_tasks.md#speed-up-tests-rake-tasks-and-migrations), -the test database can get into a corrupted state. Trying to run the migration or -dropping/resetting the test database has no effect. - -```shell -$ bundle exec spring rspec some_spec.rb -... -Failure/Error: ActiveRecord::Migration.maintain_test_schema! - -ActiveRecord::PendingMigrationError: - - - Migrations are pending. To resolve this issue, run: - - bin/rake db:migrate RAILS_ENV=test -# ~/.rvm/gems/ruby-2.3.3/gems/activerecord-4.2.10/lib/active_record/migration.rb:392:in `check_pending!' -... -0 examples, 0 failures, 1 error occurred outside of examples -``` - -To resolve, you can kill the spring server and app that lives between spec runs. - -```shell -$ ps aux | grep spring -eric 87304 1.3 2.9 3080836 482596 ?? Ss 10:12AM 4:08.36 spring app | gitlab | started 6 hours ago | test mode -eric 37709 0.0 0.0 2518640 7524 s006 S Wed11AM 0:00.79 spring server | gitlab | started 29 hours ago -$ kill 87304 -$ kill 37709 -``` - -### db:migrate `database version is too old to be migrated` error - -Users receive this error when `db:migrate` detects that the current schema version -is older than the `MIN_SCHEMA_VERSION` defined in the `Gitlab::Database` library -module. - -Over time we cleanup/combine old migrations in the codebase, so it is not always -possible to migrate GitLab from every previous version. - -In some cases you may want to bypass this check. For example, if you were on a version -of GitLab schema later than the `MIN_SCHEMA_VERSION`, and then rolled back the -to an older migration, from before. In this case, to migrate forward again, -you should set the `SKIP_SCHEMA_VERSION_CHECK` environment variable. - -```shell -bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/database_query_comments.md b/doc/development/database_query_comments.md index 2798071bc06..7f9def7e567 100644 --- a/doc/development/database_query_comments.md +++ b/doc/development/database_query_comments.md @@ -1,62 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/database_query_comments.md' +remove_date: '2022-11-05' --- -# Database query comments with Marginalia +This document was moved to [another location](database/database_query_comments.md). -The [Marginalia gem](https://github.com/basecamp/marginalia) is used to add -query comments containing application related context information to PostgreSQL -queries generated by ActiveRecord. - -It is very useful for tracing problematic queries back to the application source. - -An engineer during an on-call incident has the full context of a query -and its application source from the comments. - -## Metadata information in comments - -Queries generated from **Rails** include the following metadata in comments: - -- `application` -- `correlation_id` -- `endpoint_id` -- `line` - -Queries generated from **Sidekiq** workers include the following metadata -in comments: - -- `application` -- `jid` -- `correlation_id` -- `endpoint_id` -- `line` - -`endpoint_id` is a single field that can represent any endpoint in the application: - -- For Rails controllers, it's the controller and action. For example, `Projects::BlobController#show`. -- For Grape API endpoints, it's the route. For example, `/api/:version/users/:id`. -- For Sidekiq workers, it's the worker class name. For example, `UserStatusCleanup::BatchWorker`. - -`line` is not present in production logs due to the additional overhead required. - -Examples of queries with comments: - -- Rails: - - ```sql - /*application:web,controller:blob,action:show,correlation_id:01EZVMR923313VV44ZJDJ7PMEZ,endpoint_id:Projects::BlobController#show*/ SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 75 AND "routes"."source_type" = 'Namespace' LIMIT 1 - ``` - -- Grape: - - ```sql - /*application:web,correlation_id:01EZVN0DAYGJF5XHG9N4VX8FAH,endpoint_id:/api/:version/users/:id*/ SELECT COUNT(*) FROM "users" INNER JOIN "user_follow_users" ON "users"."id" = "user_follow_users"."followee_id" WHERE "user_follow_users"."follower_id" = 1 - ``` - -- Sidekiq: - - ```sql - /*application:sidekiq,correlation_id:df643992563683313bc0a0288fb55e23,jid:15fbc506590c625d7664b074,endpoint_id:UserStatusCleanup::BatchWorker,line:/app/workers/user_status_cleanup/batch_worker.rb:19:in `perform'*/ SELECT $1 AS one FROM "user_statuses" WHERE "user_statuses"."clear_status_at" <= $2 LIMIT $3 - ``` +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 2b215190e6d..2decd304103 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -113,6 +113,7 @@ the following preparations into account. - Ensure `db/structure.sql` is updated as [documented](migration_style_guide.md#schema-changes), and additionally ensure that the relevant version files under `db/schema_migrations` were added or removed. +- Ensure that the Database Dictionary is updated as [documented](database/database_dictionary.md). - Make migrations reversible by using the `change` method or include a `down` method when using `up`. - Include either a rollback procedure or describe how to rollback changes. - Add the output of both migrating (`db:migrate`) and rolling back (`db:rollback`) for all migrations into the MR description. @@ -179,7 +180,7 @@ Include in the MR description: - [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com): Paste both the plan and the query used in the form. - When providing query plans, make sure it hits enough data: - You can use a GitLab production replica to test your queries on a large scale, - through the `#database-lab` Slack channel or through [ChatOps](understanding_explain_plans.md#chatops). + through the `#database-lab` Slack channel or through [ChatOps](database/understanding_explain_plans.md#chatops). - Usually, the `gitlab-org` namespace (`namespace_id = 9970`) and the `gitlab-org/gitlab-foss` (`project_id = 13083`) or the `gitlab-org/gitlab` (`project_id = 278964`) projects provide enough data to serve as a good example. @@ -187,7 +188,7 @@ Include in the MR description: - If your queries belong to a new feature in GitLab.com and thus they don't return data in production: - You may analyze the query and to provide the plan from a local environment. - `#database-lab` and [postgres.ai](https://postgres.ai/) both allow updates to data (`exec UPDATE issues SET ...`) and creation of new tables and columns (`exec ALTER TABLE issues ADD COLUMN ...`). - - More information on how to find the number of actual returned records in [Understanding EXPLAIN plans](understanding_explain_plans.md) + - More information on how to find the number of actual returned records in [Understanding EXPLAIN plans](database/understanding_explain_plans.md) - For query changes, it is best to provide both the SQL queries along with the plan _before_ and _after_ the change. This helps spot differences quickly. - Include data that shows the performance improvement, preferably in @@ -200,7 +201,7 @@ Include in the MR description: #### Preparation when adding tables -- Order columns based on the [Ordering Table Columns](ordering_table_columns.md) guidelines. +- Order columns based on the [Ordering Table Columns](database/ordering_table_columns.md) guidelines. - Add foreign keys to any columns pointing to data in other tables, including [an index](migration_style_guide.md#adding-foreign-key-constraints). - Add indexes for fields that are used in statements such as `WHERE`, `ORDER BY`, `GROUP BY`, and `JOIN`s. - New tables and columns are not necessarily risky, but over time some access patterns are inherently @@ -225,7 +226,7 @@ Include in the MR description: - Consider [access patterns and data layout](database/layout_and_access_patterns.md) if new tables or columns are added. - Review migrations follow [database migration style guide](migration_style_guide.md), for example - - [Check ordering of columns](ordering_table_columns.md) + - [Check ordering of columns](database/ordering_table_columns.md) - [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints) - Ensure that migrations execute in a transaction or only contain concurrent index/foreign key helpers (with transactions disabled) @@ -247,16 +248,16 @@ Include in the MR description: - Making numerous SQL queries per record in a dataset. - Review queries (for example, make sure batch sizes are fine) - Because execution time can be longer than for a regular migration, - it's suggested to treat background migrations as post migrations: - place them in `db/post_migrate` instead of `db/migrate`. Keep in mind - that post migrations are executed post-deployment in production. + it's suggested to treat background migrations as + [post migrations](migration_style_guide.md#choose-an-appropriate-migration-type): + place them in `db/post_migrate` instead of `db/migrate`. - If a migration [has tracking enabled](database/background_migrations.md#background-jobs-tracking), ensure `mark_all_as_succeeded` is called even if no work is done. - Check [timing guidelines for migrations](migration_style_guide.md#how-long-a-migration-should-take) - Check migrations are reversible and implement a `#down` method - Check new table migrations: - Are the stated access patterns and volume reasonable? Do the assumptions they're based on seem sound? Do these patterns pose risks to stability? - - Are the columns [ordered to conserve space](ordering_table_columns.md)? + - Are the columns [ordered to conserve space](database/ordering_table_columns.md)? - Are there foreign keys for references to other tables? - Check data migrations: - Establish a time estimate for execution on GitLab.com. @@ -267,10 +268,10 @@ Include in the MR description: - Check for any overly complex queries and queries the author specifically points out for review (if any) - If not present, ask the author to provide SQL queries and query plans - (for example, by using [ChatOps](understanding_explain_plans.md#chatops) or direct + (for example, by using [ChatOps](database/understanding_explain_plans.md#chatops) or direct database access) - For given queries, review parameters regarding data distribution - - [Check query plans](understanding_explain_plans.md) and suggest improvements + - [Check query plans](database/understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) - - General guideline is for queries to come in below [100ms execution time](query_performance.md#timing-guidelines-for-queries) + - General guideline is for queries to come in below [100ms execution time](database/query_performance.md#timing-guidelines-for-queries) - Avoid N+1 problems and minimize the [query count](merge_request_performance_guidelines.md#query-counts). diff --git a/doc/development/db_dump.md b/doc/development/db_dump.md index f2076cbc410..c632302329a 100644 --- a/doc/development/db_dump.md +++ b/doc/development/db_dump.md @@ -1,56 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/db_dump.md' +remove_date: '2022-11-06' --- -# Importing a database dump into a staging environment +This document was moved to [another location](database/db_dump.md). -Sometimes it is useful to import the database from a production environment -into a staging environment for testing. The procedure below assumes you have -SSH and `sudo` access to both the production environment and the staging VM. - -**Destroy your staging VM** when you are done with it. It is important to avoid -data leaks. - -On the staging VM, add the following line to `/etc/gitlab/gitlab.rb` to speed up -large database imports. - -```shell -# On STAGING -echo "postgresql['checkpoint_segments'] = 64" | sudo tee -a /etc/gitlab/gitlab.rb -sudo touch /etc/gitlab/skip-auto-reconfigure -sudo gitlab-ctl reconfigure -sudo gitlab-ctl stop puma -sudo gitlab-ctl stop sidekiq -``` - -Next, we let the production environment stream a compressed SQL dump to our -local machine via SSH, and redirect this stream to a `psql` client on the staging -VM. - -```shell -# On LOCAL MACHINE -ssh -C gitlab.example.com sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_dump -Cc gitlabhq_production |\ - ssh -C staging-vm sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -d template1 -``` - -## Recreating directory structure - -If you need to re-create some directory structure on the staging server you can -use this procedure. - -First, on the production server, create a list of directories you want to -re-create. - -```shell -# On PRODUCTION -(umask 077; sudo find /var/opt/gitlab/git-data/repositories -maxdepth 1 -type d -print0 > directories.txt) -``` - -Copy `directories.txt` to the staging server and create the directories there. - -```shell -# On STAGING -sudo -u git xargs -0 mkdir -p < directories.txt -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md index 4e1d2e22e78..f0364f60d38 100644 --- a/doc/development/deprecation_guidelines/index.md +++ b/doc/development/deprecation_guidelines/index.md @@ -4,10 +4,10 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Deprecation guidelines +# Deprecating GitLab features -This page includes information about how and when to remove or make [breaking -changes](../contributing/index.md#breaking-changes) to GitLab features. +This page includes information about how and when to remove or make breaking changes +to GitLab features. ## Terminology @@ -37,6 +37,16 @@ changes](../contributing/index.md#breaking-changes) to GitLab features.  +**Breaking change**: + +A "breaking change" is any change that requires users to make a corresponding change to their code, settings, or workflow. "Users" might be humans, API clients, or even code classes that "use" another class. Examples of breaking changes include: + +- Removing a user-facing feature without a replacement/workaround. +- Changing the definition of an existing API (by doing things like re-naming query parameters or changing routes). +- Removing a public method from a code class. + +A breaking change can be considered major if it affects many users, or represents a significant change in behavior. + ## When can a feature be deprecated? Deprecations should be announced on the [Deprecated feature removal schedule](../../update/deprecations.md). @@ -45,6 +55,12 @@ Do not include the deprecation announcement in the merge request that introduces Use a separate MR to create a deprecation entry. For steps to create a deprecation entry, see [Deprecations](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations). +## How are Community Contributions to a deprecated feature handled? + +Development on deprecated features is restricted to Priority 1 / Severity 1 bug fixes. Any community contributions to deprecated features are unlikely to be prioritized during milestone planning. + +However, at GitLab, we [give agency](https://about.gitlab.com/handbook/values/#give-agency) to our team members. So, a member of the team associated with the contribution may decide to review and merge it at their discretion. + ## When can a feature be removed/changed? Generally, feature or configuration can be removed/changed only on major release. diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 116071cdfd9..f49d024095d 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -73,14 +73,14 @@ In this example, we have the following hypothetical values: - `driver`: the driver such a Jaeger. - `param_name`, `param_value`: these are driver specific configuration values. Configuration - parameters for Jaeger are documented [further on in this - document](#2-configure-the-gitlab_tracing-environment-variable) they should be URL encoded. + parameters for Jaeger are documented [further on in this document](#2-configure-the-gitlab_tracing-environment-variable) + they should be URL encoded. Multiple values should be separated by `&` characters like a URL. ## Using Jaeger in the GitLab Development Kit -The first tracing implementation that GitLab supports is Jaeger, and the [GitLab Development -Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) supports distributed tracing with +The first tracing implementation that GitLab supports is Jaeger, and the +[GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) supports distributed tracing with Jaeger out-of-the-box. The easiest way to access tracing from a GDK environment is through the @@ -116,8 +116,8 @@ Jaeger has many configuration options, but is very easy to start in an "all-in-o memory for trace storage (and is therefore non-persistent). The main advantage of "all-in-one" mode being ease of use. -For more detailed configuration options, refer to the [Jaeger -documentation](https://www.jaegertracing.io/docs/1.9/getting-started/). +For more detailed configuration options, refer to the +[Jaeger documentation](https://www.jaegertracing.io/docs/1.9/getting-started/). #### Using Docker @@ -201,8 +201,8 @@ If `GITLAB_TRACING` is not configured correctly, this issue is logged: ``` By default, GitLab ships with the Jaeger tracer, but other tracers can be included at compile time. -Details of how this can be done are included in the [LabKit tracing -documentation](https://pkg.go.dev/gitlab.com/gitlab-org/labkit/tracing). +Details of how this can be done are included in the +[LabKit tracing documentation](https://pkg.go.dev/gitlab.com/gitlab-org/labkit/tracing). If no log messages about tracing are emitted, the `GITLAB_TRACING` environment variable is likely not set. diff --git a/doc/development/documentation/restful_api_styleguide.md b/doc/development/documentation/restful_api_styleguide.md index 92c34c01e5d..bf1461a810d 100644 --- a/doc/development/documentation/restful_api_styleguide.md +++ b/doc/development/documentation/restful_api_styleguide.md @@ -5,7 +5,7 @@ group: unassigned description: 'Writing styles, markup, formatting, and other standards for the GitLab RESTful APIs.' --- -# RESTful API +# Documenting REST API resources REST API resources are documented in Markdown under [`/doc/api`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc/api). Each @@ -104,6 +104,9 @@ for the section. For example: > `widget_message` [introduced](<link-to-issue>) in GitLab 14.3. ``` +If the API or attribute is deployed behind a feature flag, +[include the feature flag information](feature_flags.md) in the version history. + ## Deprecations To document the deprecation of an API endpoint, follow the steps to diff --git a/doc/development/documentation/site_architecture/deployment_process.md b/doc/development/documentation/site_architecture/deployment_process.md index 5203ca52922..5f6076f3195 100644 --- a/doc/development/documentation/site_architecture/deployment_process.md +++ b/doc/development/documentation/site_architecture/deployment_process.md @@ -142,6 +142,20 @@ graph LR B--"Unpacked documentation uploaded"-->C ``` +### Manually deploy to production + +GitLab Docs is deployed to production whenever the `Build docs.gitlab.com every 4 hours` scheduled pipeline runs. By +default, this pipeline runs every four hours. + +Maintainers can [manually](../../../ci/pipelines/schedules.md#run-manually) run this pipeline to force a deployment to +production: + +1. Go to the [scheduled pipelines](https://gitlab.com/gitlab-org/gitlab-docs/-/pipeline_schedules) for `gitlab-docs`. +1. Next to `Build docs.gitlab.com every 4 hours`, select **Play** (**{play}**). + +The updated documentation is available in production after the `pages` and `pages:deploy` jobs +complete in the new pipeline. + ## Docker files The [`dockerfiles` directory](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/) contains all needed @@ -150,10 +164,10 @@ Dockerfiles to build and deploy <https://docs.gitlab.com>. It is heavily inspire | Dockerfile | Docker image | Description | |:---------------------------------------------------------------------------------------------------------------------------|:------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`Dockerfile.bootstrap`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/Dockerfile.bootstrap) | `gitlab-docs:bootstrap` | Contains all the dependencies that are needed to build the website. If the gems are updated and `Gemfile{,.lock}` changes, the image must be rebuilt. | -| [`Dockerfile.builder.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/Dockerfile.builder.onbuild) | `gitlab-docs:builder-onbuild` | Base image to build the docs website. It uses `ONBUILD` to perform all steps and depends on `gitlab-docs:bootstrap`. | -| [`Dockerfile.nginx.onbuild`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/Dockerfile.nginx.onbuild) | `gitlab-docs:nginx-onbuild` | Base image to use for building documentation archives. It uses `ONBUILD` to perform all required steps to copy the archive, and relies upon its parent `Dockerfile.builder.onbuild` that is invoked when building single documentation archives (see the `Dockerfile` of each branch) | -| [`Dockerfile.archives`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/Dockerfile.archives) | `gitlab-docs:archives` | Contains all the versions of the website in one archive. It copies all generated HTML files from every version in one location. | +| [`bootstrap.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/bootstrap.Dockerfile) | `gitlab-docs:bootstrap` | Contains all the dependencies that are needed to build the website. If the gems are updated and `Gemfile{,.lock}` changes, the image must be rebuilt. | +| [`builder.onbuild.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/builder.onbuild.Dockerfile) | `gitlab-docs:builder-onbuild` | Base image to build the docs website. It uses `ONBUILD` to perform all steps and depends on `gitlab-docs:bootstrap`. | +| [`nginx.onbuild.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/nginx.onbuild.Dockerfile) | `gitlab-docs:nginx-onbuild` | Base image to use for building documentation archives. It uses `ONBUILD` to perform all required steps to copy the archive, and relies upon its parent `Dockerfile.builder.onbuild` that is invoked when building single documentation archives (see the `Dockerfile` of each branch) | +| [`archives.Dockerfile`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/dockerfiles/archives.Dockerfile) | `gitlab-docs:archives` | Contains all the versions of the website in one archive. It copies all generated HTML files from every version in one location. | ### How to build the images diff --git a/doc/development/documentation/site_architecture/folder_structure.md b/doc/development/documentation/site_architecture/folder_structure.md index 0e8065d794f..7f29d3fba9e 100644 --- a/doc/development/documentation/site_architecture/folder_structure.md +++ b/doc/development/documentation/site_architecture/folder_structure.md @@ -85,7 +85,7 @@ place for it. Do not include the same information in multiple places. [Link to a single source of truth instead.](../styleguide/index.md#link-instead-of-repeating-text) -For example, if you have code in a repository other than the [primary repositories](index.md#architecture), +For example, if you have code in a repository other than the [primary repositories](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/architecture.md), and documentation in the same repository, you can keep the documentation in that repository. Then you can either: diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index e1e0da03abc..05e697869b9 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -22,7 +22,7 @@ At the highest level, our global nav is workflow-based. Navigation needs to help The levels under each of the higher workflow-based topics are the names of features. For example: -**Use GitLab** (_workflow_) **> Build your application** (_workflow_) **> CI/CD** (_feature_) **> Pipelines** (_feature) +**Use GitLab** (_workflow_) **> Build your application** (_workflow_) **> CI/CD** (_feature_) **> Pipelines** (_feature_) ## Choose the right words for your navigation entry @@ -39,20 +39,36 @@ as helpful as **Get started with runners**. ## Add a navigation entry -All topics should be included in the left nav. - To add a topic to the global nav, edit [`navigation.yaml`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/content/_data/navigation.yaml) and add your item. -All new pages need a navigation item. Without a navigation, the page becomes "orphaned." That -is: +Without a navigation entry: + +- The navigation closes when the page is opened, and the reader loses their place. +- The page isn't visible in a group with other pages. + +### Pages you don't need to add + +Exclude these pages from the global nav: + +- Legal notices. +- Pages in the `architecture/blueprints` directory. +- Pages in the `user/application_security/dast/checks/` directory. + +The following pages should probably be in the global nav, but the technical writers +do not actively work to add them: + +- Pages in the `/development` directory. +- Pages authored by the support team, which are under the `doc/administration/troubleshooting` directory. + +Sometimes pages for deprecated features are not in the global nav, depending on how long ago the feature was deprecated. -- The navigation shuts when the page is opened, and the reader loses their place. -- The page doesn't belong in a group with other pages. +All other pages should be in the global nav. -This means the decision to create a new page is a decision to create new navigation item and vice -versa. +The technical writing team runs a report to determine which pages are not in the nav. +For now this report is manual, but [an issue exists](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/1212) +to automate it. ### Where to add @@ -283,7 +299,7 @@ The [layout](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/layouts/global_ is fed by the [data file](#data-file), builds the global nav, and is rendered by the [default](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/layouts/default.html) layout. -The global nav contains links from all [four upstream projects](index.md#architecture). +The global nav contains links from all [four upstream projects](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/architecture.md). The [global nav URL](#urls) has a different prefix depending on the documentation file you change. | Repository | Link prefix | Final URL | diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index af24fbe303b..2864bbe7404 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -11,247 +11,30 @@ the repository which is used to generate the GitLab documentation website and is deployed to <https://docs.gitlab.com>. It uses the [Nanoc](https://nanoc.app/) static site generator. -## Architecture +View the [`gitlab-docs` architecture page](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/architecture.md) +for more information. -While the source of the documentation content is stored in the repositories for -each GitLab product, the source that is used to build the documentation -site _from that content_ is located at <https://gitlab.com/gitlab-org/gitlab-docs>. +## Documentation in other repositories -The following diagram illustrates the relationship between the repositories -from where content is sourced, the `gitlab-docs` project, and the published output. - -```mermaid - graph LR - A[gitlab-org/gitlab/doc] - B[gitlab-org/gitlab-runner/docs] - C[gitlab-org/omnibus-gitlab/doc] - D[gitlab-org/charts/gitlab/doc] - E[gitlab-org/cloud-native/gitlab-operator/doc] - Y[gitlab-org/gitlab-docs] - A --> Y - B --> Y - C --> Y - D --> Y - E --> Y - Y -- Build pipeline --> Z - Z[docs.gitlab.com] - M[//ee/] - N[//runner/] - O[//omnibus/] - P[//charts/] - Q[//operator/] - Z --> M - Z --> N - Z --> O - Z --> P - Z --> Q -``` - -GitLab docs content isn't kept in the `gitlab-docs` repository. -All documentation files are hosted in the respective repository of each -product, and all together are pulled to generate the docs website: - -- [GitLab](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) -- [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/tree/master/doc) -- [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/-/tree/main/docs) -- [GitLab Chart](https://gitlab.com/gitlab-org/charts/gitlab/-/tree/master/doc) -- [GitLab Operator](https://gitlab.com/gitlab-org/cloud-native/gitlab-operator/-/tree/master/doc) - -Learn more about [the docs folder structure](folder_structure.md). - -### Documentation in other repositories - -If you have code and documentation in a repository other than the [primary repositories](#architecture), +If you have code and documentation in a repository other than the [primary repositories](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/architecture.md), you should keep the documentation with the code in that repository. -Then you can either: +Then you can use one of these approaches: -- [Add the repository to the list of products](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/development.md#add-a-new-product) - published at <https://docs.gitlab.com>. -- [Add an entry in the global navigation](global_nav.md#add-a-navigation-entry) for - <https://docs.gitlab.com> that links to the documentation in that repository. +- Recommended. [Add the repository to the list of products](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/development.md#add-a-new-product) + published at <https://docs.gitlab.com>. The source of the documentation pages remains + in the external repository, but the resulting pages are indexed and searchable on <https://docs.gitlab.com>. +- Recommended. [Add an entry in the global navigation](global_nav.md#add-a-navigation-entry) for + <https://docs.gitlab.com> that links directly to the documentation in that external repository. + The documentation pages are not indexed or searchable on <https://docs.gitlab.com>. View [an example](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/fedb6378a3c92274ba3b6031df0d34455594e4cc/content/_data/navigation.yaml#L2944-L2946). - -## Assets - -To provide an optimized site structure, design, and a search-engine friendly -website, along with a discoverable documentation, we use a few assets for -the GitLab Documentation website. - -### External libraries - -GitLab Docs is built with a combination of external: - -- [JavaScript libraries](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/package.json). -- [Ruby libraries](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/Gemfile). - -### SEO - -- [Schema.org](https://schema.org/) -- [Google Analytics](https://marketingplatform.google.com/about/analytics/) -- [Google Tag Manager](https://developers.google.com/tag-platform/tag-manager) - -## Global navigation - -Read through [the global navigation documentation](global_nav.md) to understand: - -- How the global navigation is built. -- How to add new navigation items. - -<!-- -## Helpers - -TBA ---> - -## Pipelines - -The pipeline in the `gitlab-docs` project: - -- Tests changes to the docs site code. -- Builds the Docker images used in various pipeline jobs. -- Builds and deploys the docs site itself. -- Generates the review apps when the `review-docs-deploy` job is triggered. - -### Rebuild the docs site Docker images - -Once a week on Mondays, a scheduled pipeline runs and rebuilds the Docker images -used in various pipeline jobs, like `docs-lint`. The Docker image configuration files are -located in the [Dockerfiles directory](https://gitlab.com/gitlab-org/gitlab-docs/-/tree/main/dockerfiles). - -If you need to rebuild the Docker images immediately (must have maintainer level permissions): - -WARNING: -If you change the Dockerfile configuration and rebuild the images, you can break the main -pipeline in the main `gitlab` repository as well as in `gitlab-docs`. Create an image with -a different name first and test it to ensure you do not break the pipelines. - -1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI/CD > Pipelines**. -1. Select **Run pipeline**. -1. See that a new pipeline is running. The jobs that build the images are in the first - stage, `build-images`. You can select the pipeline number to see the larger pipeline - graph, or select the first (`build-images`) stage in the mini pipeline graph to - expose the jobs that build the images. -1. Select the **play** (**{play}**) button next to the images you want to rebuild. - - Normally, you do not need to rebuild the `image:gitlab-docs-base` image, as it - rarely changes. If it does need to be rebuilt, be sure to only run `image:docs-lint` - after it is finished rebuilding. - -### Deploy the docs site - -Every four hours a scheduled pipeline builds and deploys the docs site. The pipeline -fetches the current docs from the main project's main branch, builds it with Nanoc -and deploys it to <https://docs.gitlab.com>. - -To build and deploy the site immediately (must have the Maintainer role): - -1. In [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs), go to **{rocket}** **CI/CD > Schedules**. -1. For the `Build docs.gitlab.com every 4 hours` scheduled pipeline, select the **play** (**{play}**) button. - -Read more about [documentation deployments](deployment_process.md). - -## Using YAML data files - -The easiest way to achieve something similar to -[Jekyll's data files](https://jekyllrb.com/docs/datafiles/) in Nanoc is by -using the [`@items`](https://nanoc.app/doc/reference/variables/#items-and-layouts) -variable. - -The data file must be placed inside the `content/` directory and then it can -be referenced in an ERB template. - -Suppose we have the `content/_data/versions.yaml` file with the content: - -```yaml -versions: - - 10.6 - - 10.5 - - 10.4 -``` - -We can then loop over the `versions` array with something like: - -```erb -<% @items['/_data/versions.yaml'][:versions].each do | version | %> - -<h3><%= version %></h3> - -<% end &> -``` - -Note that the data file must have the `yaml` extension (not `yml`) and that -we reference the array with a symbol (`:versions`). - -## Archived documentation banner - -A banner is displayed on archived documentation pages with the text `This is archived documentation for -GitLab. Go to the latest.` when either: - -- The version of the documentation displayed is not the first version entry in `online` in - `content/_data/versions.yaml`. -- The documentation was built from the default branch (`main`). - -For example, if the `online` entries for `content/_data/versions.yaml` are: - -```yaml -online: - - "14.4" - - "14.3" - - "14.2" -``` - -In this case, the archived documentation banner isn't displayed: - -- For 14.4, the docs built from the `14.4` branch. The branch name is the first entry in `online`. -- For 14.5-pre, the docs built from the default project branch (`main`). - -The archived documentation banner is displayed: - -- For 14.3. -- For 14.2. -- For any other version. - -## Bumping versions of CSS and JavaScript - -Whenever the custom CSS and JavaScript files under `content/assets/` change, -make sure to bump their version in the front matter. This method guarantees that -your changes take effect by clearing the cache of previous files. - -Always use Nanoc's way of including those files, do not hardcode them in the -layouts. For example use: - -```erb -<script async type="application/javascript" src="<%= @items['/assets/javascripts/badges.*'].path %>"></script> - -<link rel="stylesheet" href="<%= @items['/assets/stylesheets/toc.*'].path %>"> -``` - -The links pointing to the files should be similar to: - -```erb -<%= @items['/path/to/assets/file.*'].path %> -``` - -Nanoc then builds and renders those links correctly according with what's -defined in [`Rules`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/Rules). - -## Linking to source files - -A helper called [`edit_on_gitlab`](https://gitlab.com/gitlab-org/gitlab-docs/blob/main/lib/helpers/edit_on_gitlab.rb) can be used -to link to a page's source file. We can link to both the simple editor and the -web IDE. Here's how you can use it in a Nanoc layout: - -- Default editor: `<a href="<%= edit_on_gitlab(@item, editor: :simple) %>">Simple editor</a>` -- Web IDE: `<a href="<%= edit_on_gitlab(@item, editor: :webide) %>">Web IDE</a>` - -If you don't specify `editor:`, the simple one is used by default. - -## Algolia search engine - -The docs site uses [Algolia DocSearch](https://community.algolia.com/docsearch/) -for its search function. - -Learn more in <https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/doc/docsearch.md>. +- Create a landing page for the product in the `gitlab` repository, and add the landing page + [to the global navigation](global_nav.md#add-a-navigation-entry), but keep the rest + of the documentation in the external repository. The landing page is indexed and + searchable on <https://docs.gitlab.com>, but the rest of the documentation is not. + For example, the [GitLab Workflow extension for VS Code](../../../user/project/repository/vscode.md). + We do not encourage the use of [pages with lists of links](../structure.md#topics-and-resources-pages), + so only use this option if the recommended options are not feasible. ## Monthly release process (versions) @@ -260,5 +43,5 @@ For more information, read about the [monthly release process](https://gitlab.co ## Review Apps for documentation merge requests -If you are contributing to GitLab docs read how to [create a Review App with each -merge request](../index.md#previewing-the-changes-live). +If you are contributing to GitLab docs read how to +[create a Review App with each merge request](../index.md#previewing-the-changes-live). diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index a02046d4466..a5d1290a17a 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -148,35 +148,60 @@ Avoid these heading titles: ## Troubleshooting -Troubleshooting can be one of two categories: +Troubleshooting topics should be the last topics on a page. -- **Troubleshooting task.** This information is written the same way as a [standard task](#task). +Troubleshooting can be one of three categories: + +- **An introductory topic.** This topic introduces the troubleshooting section of a page. + For example: + + ```markdown + ## Troubleshooting + + When working with <x feature>, you might encounter the following issues. + ``` + +- **Troubleshooting task.** The title should be similar to a [standard task](#task). For example, "Run debug tools" or "Verify syntax." -- **Troubleshooting reference.** This information has a specific format. -Troubleshooting reference information should be in this format: +- **Troubleshooting reference.** This information includes the error message. For example: -```markdown -# Title (the error message or a description of it) + ```markdown + ### The error message or a description of it -You might get an error that states <error message>. + You might get an error that states <error message>. -This issue occurs when... + This issue occurs when... -The workaround is... -``` + The workaround is... + ``` -If multiple causes or workarounds exist, consider putting them into a table format. + If multiple causes or workarounds exist, consider putting them into a table format. + If you use the exact error message, surround it in backticks so it's styled as code. + +If a page has more than five troubleshooting topics, put the content on a separate page that has troubleshooting information exclusively. Name the page `Troubleshooting <featurename>`. ### Troubleshooting headings -For the heading: +For the heading of a **Troubleshooting reference** topic: - Consider including at least a partial error message. - Use fewer than 70 characters. If you do not put the full error in the title, include it in the body text. +### Related topics + +If inline links are not sufficient, you can create a topic called **Related topics** +and include an unordered list of related topics. This topic should be above the Troubleshooting section. + +```markdown +# Related topics + +- [Configure your pipeline](link-to-topic) +- [Trigger a pipeline manually](link-to-topic) +``` + ## General heading text guidelines In general, for heading text: @@ -272,18 +297,6 @@ If you need to add more than one task, consider using subsections for each distinct task. ``` -### Related topics - -If inline links are not sufficient, you can create a topic called **Related topics** -and include an unordered list of related topics. This topic should be above the Troubleshooting section. - -```markdown -# Related topics - -- [Configure your pipeline](link-to-topic) -- [Trigger a pipeline manually](link-to-topic) -``` - ### Topics and resources pages This page has a list of links that point to important sections diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 1af0cb72055..709e6b2d0d9 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -150,6 +150,8 @@ the page is rendered to HTML. There can be only **one** level 1 heading per page - For each subsection, increment the heading level. In other words, increment the number of `#` characters in front of the heading. +- Avoid headings greater than `H5` (`#####`). If you need more than five heading levels, move the topics to a new page instead. + Headings greater than `H5` do not display in the right sidebar navigation. - Do not skip a level. For example: `##` > `####`. - Leave one blank line before and after the heading. @@ -162,6 +164,13 @@ Also, do not use links as part of heading text. See also [heading guidelines for specific topic types](../structure.md). +### Backticks in Markdown + +Use backticks for: + +- [Code blocks](#code-blocks). +- Error messages. + ### Markdown Rules GitLab ensures that the Markdown used across all documentation is consistent, as @@ -722,10 +731,12 @@ We include guidance for links in these categories: - Use inline link Markdown markup `[Text](https://example.com)`. It's easier to read, review, and maintain. Do not use `[Text][identifier]` reference-style links. - - Use meaningful anchor text. For example, instead of writing something like `Read more about merge requests [here](LINK)`, write `Read more about [merge requests](LINK)`. +- Put the entire link on a single line. Some of our [linters](../testing.md) do not + validate links when split over multiple lines, and incorrect or broken links could + slip through. ### Links to internal documentation @@ -787,45 +798,15 @@ section of GitLab. ### Links to external documentation -When describing interactions with external software, it's often helpful to -include links to external documentation. When possible, make sure that you're -linking to an [**authoritative** source](#authoritative-sources). For example, -if you're describing a feature in Microsoft's Active Directory, include a link -to official Microsoft documentation. - -### Authoritative sources - -When citing external information, use sources that are written by the people who -created the item or product in question. These sources are the most likely to be -accurate and remain up to date. - -Examples of authoritative sources include: - -- Specifications, such as a [Request for Comments](https://www.ietf.org/standards/rfcs/) - document from the Internet Engineering Task Force. -- Official documentation for a product. For example, if you're setting up an - interface with the Google OAuth 2 authorization server, include a link to - Google's documentation. -- Official documentation for a project. For example, if you're citing NodeJS - functionality, refer directly to [NodeJS documentation](https://nodejs.org/en/docs/). -- Books from an authoritative publisher. +When possible, avoid links to external documentation. These links can easily become outdated, and are difficult to maintain. -Examples of sources to avoid include: +- [They lead to link rot](https://en.wikipedia.org/wiki/Link_rot). +- [They create issues with maintenance](https://gitlab.com/gitlab-org/gitlab/-/issues/368300). -- Personal blog posts. -- Wikipedia. -- Non-trustworthy articles. -- Discussions on forums such as Stack Overflow. -- Documentation from a company that describes another company's product. +Sometimes links are required. They might clarify troubleshooting steps or help prevent duplication of content. +Sometimes they are more precise and will be maintained more actively. -While many of these sources to avoid can help you learn skills and or features, -they can become obsolete quickly. Nobody is obliged to maintain any of these -sites. Therefore, we should avoid using them as reference literature. - -NOTE: -Non-authoritative sources are acceptable only if there is no equivalent -authoritative source. Even then, focus on non-authoritative sources that are -extensively cited or peer-reviewed. +For each external link you add, weigh the customer benefit with the maintenance difficulties. ### Links requiring permissions @@ -950,6 +931,16 @@ For example: 1. Optional. Enter a description for the job. ``` +### Recommended steps + +If a step is recommended, start the step with the word `Recommended` followed by a period. + +For example: + +```markdown +1. Recommended. Enter a description for the job. +``` + ### Documenting multiple fields at once If the UI text sufficiently explains the fields in a section, do not include a task step for every field. @@ -1106,6 +1097,36 @@ include a visual representation to help readers understand it, you can: an area of the screen. - Create a short video of the interaction and link to it. +## Emojis + +Don't use the Markdown emoji format, for example `:smile:`, for any purpose. Use +[GitLab SVG icons](#gitlab-svg-icons) instead. + +Use of emoji in Markdown requires GitLab Flavored Markdown, which is not supported by Kramdown, +the Markdown rendering engine used for GitLab documentation. + +## GitLab SVG icons + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/384) in GitLab 12.7. + +You can use icons from the [GitLab SVG library](https://gitlab-org.gitlab.io/gitlab-svgs/) +directly in the documentation. For example, `**{tanuki}**` renders as: **{tanuki}**. + +In most cases, you should avoid using the icons in text. +However, you can use an icon when hover text is the only +available way to describe a UI element. For example, **Delete** or **Edit** buttons +often have hover text only. + +When you do use an icon, start with the hover text and follow it with the SVG reference in parentheses. + +- Avoid: `Select **{pencil}** **Edit**.` This generates as: Select **{pencil}** **Edit**. +- Use instead: `Select **Edit** (**{pencil}**).` This generates as: Select **Edit** (**{pencil}**). + +Do not use words to describe the icon: + +- Avoid: `Select **Erase job log** (the trash icon).` +- Use instead: `Select **Erase job log** (**{remove}**).` This generates as: Select **Erase job log** (**{remove}**). + ## Videos Adding GitLab YouTube video tutorials to the documentation is highly @@ -1187,28 +1208,6 @@ different mobile devices. `/help`, because the GitLab Markdown processor doesn't support iframes. It's hidden on the documentation site, but is displayed by `/help`. -## GitLab SVG icons - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/384) in GitLab 12.7. - -You can use icons from the [GitLab SVG library](https://gitlab-org.gitlab.io/gitlab-svgs/) -directly in the documentation. For example, `**{tanuki}**` renders as: **{tanuki}**. - -In most cases, you should avoid using the icons in text. -However, you can use an icon when hover text is the only -available way to describe a UI element. For example, **Delete** or **Edit** buttons -often have hover text only. - -When you do use an icon, start with the hover text and follow it with the SVG reference in parentheses. - -- Avoid: `Select **{pencil}** **Edit**.` This generates as: Select **{pencil}** **Edit**. -- Use instead: `Select **Edit** (**{pencil}**).` This generates as: Select **Edit** (**{pencil}**). - -Do not use words to describe the icon: - -- Avoid: `Select **Erase job log** (the trash icon).` -- Use instead: `Select **Erase job log** (**{remove}**).` This generates as: Select **Erase job log** (**{remove}**). - ## Alert boxes Use alert boxes to call attention to information. Use them sparingly, and never have an alert box immediately follow another alert box. diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index c753c39b727..1976caefc8e 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -239,6 +239,13 @@ Use **CI/CD minutes** instead of **CI minutes**, **pipeline minutes**, **pipelin Do not use **click**. Instead, use **select** with buttons, links, menu items, and lists. **Select** applies to more devices, while **click** is more specific to a mouse. +## cloud native + +When you're talking about using a Kubernetes cluster to host GitLab, you're talking about a **cloud-native version of GitLab**. +This version is different than the larger, more monolithic **Omnibus package** that is used to deploy GitLab. + +You can also use **cloud-native GitLab** for short. It should be hyphenated and lowercase. + ## collapse Use **collapse** instead of **close** when you are talking about expanding or collapsing a section in the UI. @@ -434,6 +441,17 @@ Do not make **GitLab** possessive (GitLab's). This guidance follows [GitLab Trad **GitLab.com** refers to the GitLab instance managed by GitLab itself. +## GitLab Helm chart, GitLab chart + +To deploy a cloud-native version of GitLab, use: + +- The GitLab Helm chart (long version) +- The GitLab chart (short version) + +Do not use **the `gitlab` chart**, **the GitLab Chart**, or **the cloud-native chart**. + +You use the **GitLab Helm chart** to deploy **cloud-native GitLab** in a Kubernetes cluster. + ## GitLab Flavored Markdown When possible, spell out [**GitLab Flavored Markdown**](../../../user/markdown.md). @@ -1127,7 +1145,7 @@ in present tense, active voice. ## you, your, yours Use **you**, **your**, and **yours** instead of [**the user** and **the user's**](#user-users). -Documentation should be from the [point of view](https://design.gitlab.com/content/voice-tone#point-of-view) of the reader. +Documentation should be from the [point of view](https://design.gitlab.com/content/voice-tone/#point-of-view) of the reader. Use: diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index d55cbe28d9b..428a57a11fb 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -81,6 +81,36 @@ This requires you to either: ### Documentation link tests +Merge requests containing changes to Markdown (`.md`) files run a `docs-lint links` +job, which runs two types of link checks. In both cases, links with destinations +that begin with `http` or `https` are considered external links, and skipped: + +- `bundle exec nanoc check internal_links`: Tests links to internal pages. +- `bundle exec nanoc check internal_anchors`: Tests links to subheadings (anchors) on internal pages. + +Failures from these tests are displayed at the end of the test results in the **Issues found!** area. +For example, failures in the `internal_anchors` test follow this format: + +```plaintext +[ ERROR ] internal_anchors - Broken anchor detected! + - source file `/tmp/gitlab-docs/public/ee/user/application_security/api_fuzzing/index.html` + - destination `/tmp/gitlab-docs/public/ee/development/code_review.html` + - link `../../../development/code_review.html#review-response-slo` + - anchor `#review-response-slo` +``` + +- **Source file**: The full path to the file containing the error. To find the + file in the `gitlab` repository, replace `/tmp/gitlab-docs/public/ee` with `doc`, and `.html` with `.md`. +- **Destination**: The full path to the file not found by the test. To find the + file in the `gitlab` repository, replace `/tmp/gitlab-docs/public/ee` with `doc`, and `.html` with `.md`. +- **Link**: The actual link the script attempted to find. +- **Anchor**: If present, the subheading (anchor) the script attempted to find. + +Check for multiple instances of the same broken link on each page reporting an error. +Even if a specific broken link appears multiple times on a page, the test reports it only once. + +#### Run document link tests locally + To execute documentation link tests locally: 1. Navigate to the [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. @@ -219,12 +249,12 @@ You can use markdownlint: ### Vale -[Vale](https://docs.errata.ai/vale/about/) is a grammar, style, and word usage linter for the +[Vale](https://vale.sh/) is a grammar, style, and word usage linter for the English language. Vale's configuration is stored in the [`.vale.ini`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.vale.ini) file located in the root directory of projects. -Vale supports creating [custom tests](https://docs.errata.ai/vale/styles) that extend any of +Vale supports creating [custom tests](https://vale.sh/docs/topics/styles/) that extend any of several types of checks, which we store in the `.linting/vale/styles/gitlab` directory in the documentation directory of projects. @@ -241,7 +271,7 @@ This configuration is also used in build pipelines, where You can use Vale: -- [On the command line](https://docs.errata.ai/vale/cli). +- [On the command line](https://vale.sh/docs/vale-cli/structure/). - [In a code editor](#configure-editors). - [In a Git hook](#configure-pre-push-hooks). Vale only reports errors in the Git hook (the same configuration as the CI/CD pipelines), and does not report suggestions or warnings. @@ -250,7 +280,8 @@ You can use Vale: Vale returns three types of results: -- **Error** - For branding and trademark issues, and words or phrases with ambiguous meanings. +- **Error** - For branding guidelines, trademark guidelines, and anything that causes content on + the docs site to render incorrectly. - **Warning** - For Technical Writing team style preferences. - **Suggestion** - For basic technical writing tenets and best practices. @@ -304,7 +335,30 @@ For example, a page that scores `12` before a set of changes, and `9` after, ind general complexity level of the page. The readability score is calculated based on the number of words per sentence, and the number -of syllables per word. For more information, see [the Vale documentation](https://docs.errata.ai/vale/styles#metric). +of syllables per word. For more information, see [the Vale documentation](https://vale.sh/docs/topics/styles/#metric). + +#### When to add a new Vale rule + +It's tempting to add a Vale rule for every style guide rule. However, we should be +mindful of the effort to create and enforce a Vale rule, and the noise it creates. + +In general, follow these guidelines: + +- If you add an [error-level Vale rule](#vale-result-types), you must fix + the existing occurrences of the issue in the documentation before you can add the rule. + + If there are too many issues to fix in a single merge request, add the rule at a + `warning` level. Then, fix the existing issues in follow-up merge requests. + When the issues are fixed, promote the rule to an `error`. + +- If you add a warning-level or suggestion-level rule, consider: + + - How many more warnings or suggestions it creates in the Vale output. If the + number of additional warnings is significant, the rule might be too broad. + + - How often an author might ignore it because it's acceptable in the context. + If the rule is too subjective, it cannot be adequately enforced and creates + unnecessary additional warnings. ### Install linters @@ -399,8 +453,6 @@ To configure Vale in your editor, install one of the following as appropriate: In this setup the `markdownlint` checker is set as a "next" checker from the defined `vale` checker. Enabling this custom Vale checker provides error linting from both Vale and markdownlint. -We don't use [Vale Server](https://docs.errata.ai/vale-server/install). - ### Configure pre-push hooks Git [pre-push hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) allow Git users to: @@ -479,7 +531,7 @@ document: Whenever possible, exclude only the problematic rule and lines. For more information, see -[Vale's documentation](https://docs.errata.ai/vale/scoping#markup-based-configuration). +[Vale's documentation](https://vale.sh/docs/topics/scoping/). ### Disable markdownlint tests diff --git a/doc/development/documentation/versions.md b/doc/development/documentation/versions.md index 067c37d30aa..3679c731a77 100644 --- a/doc/development/documentation/versions.md +++ b/doc/development/documentation/versions.md @@ -182,7 +182,7 @@ GitLab supports the current major version and two previous major versions. For example, if 15.0 is the current major version, all major and minor releases of GitLab 15.0, 14.0, and 13.0 are supported. -[View the list of supported versions](https://about.gitlab.com/support/statement-of-support.html#version-support). +[View the list of supported versions](https://about.gitlab.com/support/statement-of-support/#version-support). If you see version history items or inline text that refers to unsupported versions, you can remove it. @@ -198,8 +198,8 @@ We cannot guarantee future feature work, and promises like these can raise legal issues. Instead, say that an issue exists. For example: -- Support for improvements is tracked `[in this issue](LINK)`. -- You cannot do this thing, but `[an issue exists](LINK)` to change this behavior. +- Support for improvements is proposed in issue `[issue-number](LINK-TO-ISSUE)`. +- You cannot do this thing, but issue `[issue-number](LINK-TO-ISSUE)` proposes to change this behavior. You can say that we plan to remove a feature. diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 28cf6d4e1e3..777bc77875e 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -6,8 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Guidelines for implementing Enterprise Edition features -- **Write the code and the tests.**: As with any code, EE features should have - good test coverage to prevent regressions. +- **Place code in `ee/`**: Put all Enterprise Edition (EE) inside the `ee/` top-level directory. The + rest of the code must be as close to the Community Edition (CE) files as possible. +- **Write tests**: As with any code, EE features must have good test coverage to prevent + regressions. All `ee/` code must have corresponding tests in `ee/`. - **Write documentation.**: Add documentation to the `doc/` directory. Describe the feature and include screenshots, if applicable. Indicate [what editions](documentation/styleguide/index.md#product-tier-badges) the feature applies to. @@ -16,54 +18,72 @@ info: To determine the technical writer assigned to the Stage/Group associated w [EE features list](https://about.gitlab.com/features/). <!-- markdownlint-enable MD044 --> -## Act as SaaS +## Implement a new EE feature -When developing locally, there are times when you need your instance to act like the SaaS version of the product. -In those instances, you can simulate SaaS by exporting an environment variable as seen below: +If you're developing a GitLab Starter, GitLab Premium, or GitLab Ultimate licensed feature, use these steps to +add your new feature or extend it. -```shell -export GITLAB_SIMULATE_SAAS=1 -``` +GitLab license features are added to [`ee/app/models/gitlab_subscriptions/features.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/gitlab_subscriptions/features.rb). To determine how +to modify this file, first discuss how your feature fits into our licensing with your Product Manager. -There are many ways to pass an environment variable to your local GitLab instance. -For example, you can create a `env.runit` file in the root of your GDK with the above snippet. +Use the following questions to guide you: -## Act as CE when unlicensed +1. Is this a new feature, or are you extending an existing licensed feature? + - If your feature already exists, you don't have to modify `features.rb`, but you + must locate the existing feature identifier to [guard it](#guard-your-ee-feature). + - If this is a new feature, decide on an identifier, such as `my_feature_name`, to add to the + `features.rb` file. +1. Is this a **GitLab Starter**, **GitLab Premium**, or **GitLab Ultimate** feature? + - Based on the plan you choose to use the feature in, add the feature identifier to `STARTER_FEATURES`, + `PREMIUM_FEATURES`, or `ULTIMATE_FEATURES`. +1. Will this feature be available globally (system-wide at the GitLab instance level)? + - Features such as [Geo](../administration/geo/index.md) and + [Database Load Balancing](../administration/postgresql/database_load_balancing.md) are used by the entire instance + and cannot be restricted to individual user namespaces. These features are defined in the instance license. + Add these features to `GLOBAL_FEATURES`. -Since the implementation of -[GitLab CE features to work with unlicensed EE instance](https://gitlab.com/gitlab-org/gitlab/-/issues/2500) -GitLab Enterprise Edition should work like GitLab Community Edition -when no license is active. So EE features always should be guarded by -`project.feature_available?` or `group.licensed_feature_available?` (or -`License.feature_available?` if it is a system-wide feature). +### Guard your EE feature -Frontend features should be guarded by pushing a flag from the backend by [using `push_licensed_feature`](licensed_feature_availability.md#restricting-frontend-features), and checked using `this.glFeatures.someFeature` in the frontend. For example: +A licensed feature can only be available to licensed users. You must add a check or guard +to determine if users have access to the feature. -```html -<script> -import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; +To guard your licensed feature: -export default { - mixins: [glFeatureFlagMixin()], - components: { - EEComponent: () => import('ee_component/components/test.vue'), - }, - computed: { - shouldRenderComponent() { - return this.glFeatures.myEEFeature; - } - }, -}; -</script> +1. Locate your feature identifier in `ee/app/models/gitlab_subscriptions/features.rb`. +1. Use the following methods, where `my_feature_name` is your feature + identifier: -<template> - <div> - <ee-component v-if="shouldRenderComponent"/> - </div> -</template> -``` + - In a project context: + + ```ruby + my_project.licensed_feature_available?(:my_feature_name) # true if available for my_project + ``` + + - In a group or user namespace context: + + ```ruby + my_group.licensed_feature_available?(:my_feature_name) # true if available for my_group + ``` -Look in `ee/app/models/license.rb` for the names of the licensed features. + - For a global (system-wide) feature: + + ```ruby + License.feature_available?(:my_feature_name) # true if available in this instance + ``` + +1. Optional. If your global feature is also available to namespaces with a paid plan, combine two +feature identifiers to allow both admins and group users. For example: + + ```ruby + License.feature_available?(:my_feature_name) || group.licensed_feature_available?(:my_feature_name_for_namespace) # Both admins and group members can see this EE feature + ``` + +### Simulate a CE instance when unlicensed + +After the implementation of +[GitLab CE features to work with unlicensed EE instance](https://gitlab.com/gitlab-org/gitlab/-/issues/2500) +GitLab Enterprise Edition works like GitLab Community Edition +when no license is active. CE specs should remain untouched as much as possible and extra specs should be added for EE. Licensed features can be stubbed using the @@ -74,7 +94,7 @@ setting the [`FOSS_ONLY` environment variable](https://gitlab.com/gitlab-org/git to something that evaluates as `true`. The same works for running tests (for example `FOSS_ONLY=1 yarn jest`). -### Running feature specs as CE +#### Run feature specs as CE When running [feature specs](testing_guide/best_practices.md#system--feature-tests) as CE, you should ensure that the edition of backend and frontend match. @@ -98,7 +118,28 @@ To do so: bin/rspec spec/features/<path_to_your_spec> ``` -## CI pipelines in a FOSS context +### Simulate a SaaS instance + +If you're developing locally and need your instance to act like the SaaS version of the product, +you can simulate SaaS by exporting an environment variable: + +```shell +export GITLAB_SIMULATE_SAAS=1 +``` + +There are many ways to pass an environment variable to your local GitLab instance. +For example, you can create a `env.runit` file in the root of your GDK with the above snippet. + +#### Allow use of licensed EE feature + +To enable plans per namespace turn on the `Allow use of licensed EE features` option from the settings page. +This will make licensed EE features available to projects only if the project namespace's plan includes the feature +or if the project is public. To enable it: + +1. If you are developing locally, follow the steps in [Simulate a SaaS instance](#simulate-a-saas-instance) to make the option available. +1. Visit Admin > Settings > General > "Account and limit" and enable "Allow use of licensed EE features". + +### Run CI pipelines in a FOSS context By default, merge request pipelines for development run in an EE-context only. If you are developing features that differ between FOSS and EE, you may wish to run pipelines in a @@ -108,10 +149,7 @@ To run pipelines in both contexts, add the `~"pipeline:run-as-if-foss"` label to See the [As-if-FOSS jobs](pipelines.md#as-if-foss-jobs) pipelines documentation for more information. -## Separation of EE code - -All EE code should be put inside the `ee/` top-level directory. The -rest of the code should be as close to the CE files as possible. +## Separation of EE code in the backend ### EE-only features @@ -144,7 +182,7 @@ To test an EE class that doesn't exist in CE, create the spec file as you normal would in the `ee/spec` directory, but without the second `ee/` subdirectory. For example, a class `ee/app/models/vulnerability.rb` would have its tests in `ee/spec/models/vulnerability_spec.rb`. -### EE features based on CE features +### Extend CE features with EE backend code For features that build on existing CE features, write a module in the `EE` namespace and inject it in the CE class, on the last line of the file that the @@ -243,8 +281,8 @@ There are a few gotchas with it: overriding the method, because we can't know when the overridden method (that is, calling `super` in the overriding method) would want to stop early. In this case, we shouldn't just override it, but update the original method - to make it call the other method we want to extend, like a [template method - pattern](https://en.wikipedia.org/wiki/Template_method_pattern). + to make it call the other method we want to extend, like a + [template method pattern](https://en.wikipedia.org/wiki/Template_method_pattern). For example, given this base: ```ruby @@ -633,7 +671,7 @@ might need different strategies to extend it. To apply different strategies easily, we would use `extend ActiveSupport::Concern` in the EE module. Put the EE module files following -[EE features based on CE features](#ee-features-based-on-ce-features). +[Extend CE features with EE backend code](#extend-ce-features-with-ee-backend-code). #### EE API routes @@ -1009,9 +1047,9 @@ FactoryBot.define do end ``` -## JavaScript code in `assets/javascripts/` +## Separate of EE code in the frontend -To separate EE-specific JS-files we should also move the files into an `ee` folder. +To separate EE-specific JS-files, move the files into an `ee` folder. For example there can be an `app/assets/javascripts/protected_branches/protected_branches_bundle.js` and an @@ -1032,40 +1070,123 @@ import bundle from 'ee/protected_branches/protected_branches_bundle.js'; import bundle from 'ee_else_ce/protected_branches/protected_branches_bundle.js'; ``` -See the frontend guide [performance section](fe_guide/performance.md) for -information on managing page-specific JavaScript within EE. +### Add new EE-only features in the frontend + +If the feature being developed is not present in CE, add your entry point in +`ee/`. For example: + +```shell +# Add HTML element to mount +ee/app/views/admin/geo/designs/index.html.haml + +# Init the application +ee/app/assets/javascripts/pages/ee_only_feature/index.js + +# Mount the feature +ee/app/assets/javascripts/ee_only_feature/index.js +``` + +Feature guarding `licensed_feature_available?` and `License.feature_available?` typical +occurs in the controller, as described in the [backend guide](#ee-only-features). + +#### Test EE-only features + +Add your EE tests to `ee/spec/frontend/` following the same directory structure you use for CE. + +### Extend CE features with EE frontend code + +Use the [`push_licensed_feature`](#guard-your-ee-feature) to guard frontend features that extend +existing views: + +```ruby +# ee/app/controllers/ee/admin/my_controller.rb +before_action do + push_licensed_feature(:my_feature_name) # for global features +end +``` + +```ruby +# ee/app/controllers/ee/group/my_controller.rb +before_action do + push_licensed_feature(:my_feature_name, @group) # for group pages +end +``` + +```ruby +# ee/app/controllers/ee/project/my_controller.rb +before_action do + push_licensed_feature(:my_feature_name, @group) # for group pages + push_licensed_feature(:my_feature_name, @project) # for project pages +end +``` + +Verify your feature appears in `gon.licensed_features` in the browser console. -## Vue code in `assets/javascript` +#### Extend Vue applications with EE Vue components -### script tag +EE licensed features that enhance existing functionality in the UI add new +elements or interactions to your Vue application as components. -#### Child Component only used in EE +To separate template differences, use a child EE component to separate Vue template differences. +You must import the EE component [asynchronously](https://vuejs.org/v2/guide/components-dynamic-async.html#Async-Components). -To separate Vue template differences we should [import the components asynchronously](https://vuejs.org/v2/guide/components-dynamic-async.html#Async-Components). +This allows GitLab to load the correct component in EE, while in CE GitLab loads an empty component +that renders nothing. This code **must** exist in the CE repository, in addition to the EE repository. -Doing this allows for us to load the correct component in EE while in CE -we can load a empty component that renders nothing. This code **should** -exist in the CE repository as well as the EE repository. +A CE component acts as the entry point to your EE feature. To add a EE component, +locate it the `ee/` directory and add it with `import('ee_component/...')`: ```html <script> +// app/assets/javascripts/feature/components/form.vue + export default { + mixins: [glFeatureFlagMixin()], components: { - EEComponent: () => import('ee_component/components/test.vue'), + // Import an EE component from CE + MyEeComponent: () => import('ee_component/components/my_ee_component.vue'), }, }; </script> <template> <div> - <ee-component /> + <!-- ... --> + <my-ee-component/> + <!-- ... --> </div> </template> ``` -#### For JS code that is EE only, like props, computed properties, methods, etc +Check `glFeatures` to ensure that the Vue components are guarded. The components render only when +the license is present. -- Please do not use mixins unless ABSOLUTELY NECESSARY. Please try to find an alternative pattern. +```html +<script> +// ee/app/assets/javascripts/feature/components/special_component.vue + +import glFeatureFlagMixin from '~/vue_shared/mixins/gl_feature_flags_mixin'; + +export default { + mixins: [glFeatureFlagMixin()], + computed: { + shouldRenderComponent() { + // Comes from gon.licensed_features as a camel-case version of `my_feature_name` + return this.glFeatures.myFeatureName; + } + }, +}; +</script> + +<template> + <div v-if="shouldRenderComponent"> + <!-- EE licensed feature UI --> + </div> +</template> +``` + +NOTE: +Do not use mixins unless ABSOLUTELY NECESSARY. Try to find an alternative pattern. ##### Recommended alternative approach (named/scoped slots) @@ -1138,11 +1259,65 @@ export default { **For EE components that need different results for the same computed values, we can pass in props to the CE wrapper as seen in the example.** - **EE Child components** - - Since we are using the asynchronous loading to check which component to load, we'd still use the component's name, check [this example](#child-component-only-used-in-ee). + - Since we are using the asynchronous loading to check which component to load, we'd still use the component's name, check [this example](#extend-vue-applications-with-ee-vue-components). - **EE extra HTML** - For the templates that have extra HTML in EE we should move it into a new component and use the `ee_else_ce` dynamic import +#### Extend other JS code + +To extend JS files, complete the following steps: + +1. Use the `ee_else_ce` helper, where that EE only code must be inside the `ee/` folder. + 1. Create an EE file with only the EE, and extend the CE counterpart. + 1. For code inside functions that can't be extended, move the code to a new file and use `ee_else_ce` helper: + +```javascript + import eeCode from 'ee_else_ce/ee_code'; + + function test() { + const test = 'a'; + + eeCode(); + + return test; + } +``` + +In some cases, you'll need to extend other logic in your application. To extend your JS +modules, create an EE version of the file and extend it with your custom logic: + +```javascript +// app/assets/javascripts/feature/utils.js + +export const myFunction = () => { + // ... +}; + +// ... other CE functions ... +``` + +```javascript +// ee/app/assets/javascripts/feature/utils.js +import { + myFunction as ceMyFunction, +} from '~/feature/utils'; + +/* eslint-disable import/export */ + +// Export same utils as CE +export * from '~/feature/utils'; + +// Only override `myFunction` +export const myFunction = () => { + const result = ceMyFunction(); + // add EE feature logic + return result; +}; + +/* eslint-enable import/export */ +``` + #### Testing modules using EE/CE aliases When writing Frontend tests, if the module under test imports other modules with `ee_else_ce/...` and these modules are also needed by the relevant test, then the relevant test **must** import these modules with `ee_else_ce/...`. This avoids unexpected EE or FOSS failures, and helps ensure the EE behaves like CE when it is unlicensed. @@ -1185,29 +1360,7 @@ describe('ComponentUnderTest', () => { ``` -### Non Vue Files - -For regular JS files, the approach is similar. - -1. We keep using the [`ee_else_ce`](../development/ee_features.md#javascript-code-in-assetsjavascripts) helper, this means that EE only code should be inside the `ee/` folder. - 1. An EE file should be created with the EE only code, and it should extend the CE counterpart. - 1. For code inside functions that can't be extended, the code should be moved into a new file and we should use `ee_else_ce` helper: - -#### Example - -```javascript - import eeCode from 'ee_else_ce/ee_code'; - - function test() { - const test = 'a'; - - eeCode(); - - return test; - } -``` - -## SCSS code in `assets/stylesheets` +#### SCSS code in `assets/stylesheets` If a component you're adding styles for is limited to EE, it is better to have a separate SCSS file in an appropriate directory within `app/assets/stylesheets`. @@ -1218,9 +1371,8 @@ styles are usually kept in a stylesheet that is common for both CE and EE, and i to isolate such ruleset from rest of CE rules (along with adding comment describing the same) to avoid conflicts during CE to EE merge. -### Bad - ```scss +// Bad .section-body { .section-title { background: $gl-header-color; @@ -1234,9 +1386,8 @@ to avoid conflicts during CE to EE merge. } ``` -### Good - ```scss +// Good .section-body { .section-title { background: $gl-header-color; @@ -1252,7 +1403,7 @@ to avoid conflicts during CE to EE merge. // EE-specific end ``` -## GitLab-svgs +### GitLab-svgs Conflicts in `app/assets/images/icons.json` or `app/assets/images/icons.svg` can be resolved simply by regenerating those assets with diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index d32ceb43ce9..47942817790 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -38,7 +38,7 @@ Additionally, if you need large repositories or multiple forks for testing, plea The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). -After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/topics/data-types#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). +After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/docs/manual/data-types/#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! @@ -277,8 +277,8 @@ These Advanced Search migrations, like any other GitLab changes, need to support Depending on the order of deployment, it's possible that the migration has started or finished and there's still a server running the application code from before the -migration. We need to take this into consideration until we can [ensure all Advanced Search migrations -start after the deployment has finished](https://gitlab.com/gitlab-org/gitlab/-/issues/321619). +migration. We need to take this into consideration until we can +[ensure all Advanced Search migrations start after the deployment has finished](https://gitlab.com/gitlab-org/gitlab/-/issues/321619). ### Reverting a migration @@ -317,9 +317,8 @@ safely can. We choose to use GitLab major version upgrades as a safe time to remove backwards compatibility for indices that have not been fully migrated. We -[document this in our upgrade -documentation](../update/index.md#upgrading-to-a-new-major-version). We also -choose to replace the migration code with the halted migration +[document this in our upgrade documentation](../update/index.md#upgrading-to-a-new-major-version). +We also choose to replace the migration code with the halted migration and remove tests so that: - We don't need to maintain any code that is called from our Advanced Search @@ -381,7 +380,7 @@ the volume of updates. All of the indexing happens in Sidekiq, so much of the relevant logs for the Elasticsearch integration can be found in -[`sidekiq.log`](../administration/logs.md#sidekiqlog). In particular, all +[`sidekiq.log`](../administration/logs/index.md#sidekiqlog). In particular, all Sidekiq workers that make requests to Elasticsearch in any way will log the number of requests and time taken querying/writing to Elasticsearch. This can be useful to understand whether or not your cluster is keeping up with @@ -390,26 +389,25 @@ indexing. Searching Elasticsearch is done via ordinary web workers handling requests. Any requests to load a page or make an API request, which then make requests to Elasticsearch, will log the number of requests and the time taken to -[`production_json.log`](../administration/logs.md#production_jsonlog). These +[`production_json.log`](../administration/logs/index.md#production_jsonlog). These logs will also include the time spent on Database and Gitaly requests, which may help to diagnose which part of the search is performing poorly. There are additional logs specific to Elasticsearch that are sent to -[`elasticsearch.log`](../administration/logs.md#elasticsearchlog) +[`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) that may contain information to help diagnose performance issues. ### Performance Bar -Elasticsearch requests will be displayed in the [`Performance -Bar`](../administration/monitoring/performance/performance_bar.md), which can +Elasticsearch requests will be displayed in the +[`Performance Bar`](../administration/monitoring/performance/performance_bar.md), which can be used both locally in development and on any deployed GitLab instance to diagnose poor search performance. This will show the exact queries being made, which is useful to diagnose why a search might be slow. ### Correlation ID and `X-Opaque-Id` -Our [correlation -ID](distributed_tracing.md#developer-guidelines-for-working-with-correlation-ids) +Our [correlation ID](distributed_tracing.md#developer-guidelines-for-working-with-correlation-ids) is forwarded by all requests from Rails to Elasticsearch as the [`X-Opaque-Id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html#_identifying_running_tasks) header which allows us to track any @@ -477,13 +475,13 @@ documented here in case it is useful for others. The relevant logs that could theoretically be used to figure out what needs to be replayed are: 1. All non-repository updates that were synced can be found in - [`elasticsearch.log`](../administration/logs.md#elasticsearchlog) by + [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by searching for [`track_items`](https://gitlab.com/gitlab-org/gitlab/-/blob/1e60ea99bd8110a97d8fc481e2f41cab14e63d31/ee/app/services/elastic/process_bookkeeping_service.rb#L25) and these can be replayed by sending these items again through `::Elastic::ProcessBookkeepingService.track!` 1. All repository updates that occurred can be found in - [`elasticsearch.log`](../administration/logs.md#elasticsearchlog) by + [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by searching for [`indexing_commit_range`](https://gitlab.com/gitlab-org/gitlab/-/blob/6f9d75dd3898536b9ec2fb206e0bd677ab59bd6d/ee/lib/gitlab/elastic/indexer.rb#L41). Replaying these requires resetting the @@ -492,13 +490,13 @@ theoretically be used to figure out what needs to be replayed are: the project using [`ElasticCommitIndexerWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_commit_indexer_worker.rb) 1. All project deletes that occurred can be found in - [`sidekiq.log`](../administration/logs.md#sidekiqlog) by searching for + [`sidekiq.log`](../administration/logs/index.md#sidekiqlog) by searching for [`ElasticDeleteProjectWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_delete_project_worker.rb). These updates can be replayed by triggering another `ElasticDeleteProjectWorker`. -With the above methods and taking regular [Elasticsearch -snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html) +With the above methods and taking regular +[Elasticsearch snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html) we should be able to recover from different kinds of data loss issues in a relatively short period of time compared to indexing everything from scratch. diff --git a/doc/development/emails.md b/doc/development/emails.md index a5c2789a3ea..1b3c9226dd8 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -60,7 +60,7 @@ See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html# # The email address including the %{key} placeholder that will be replaced to reference the # item being replied to. This %{key} should be included in its entirety within the email # address and not replaced by another value. - # For example: emailadress+%{key}@gmail.com. + # For example: emailaddress+%{key}@gmail.com. # The placeholder must appear in the "user" part of the address (before the `@`). It can be omitted but some features, # including Service Desk, may not work properly. address: "gitlab-incoming+%{key}@gmail.com" @@ -160,9 +160,10 @@ and Helm Chart configuration (see [example merge request](https://gitlab.com/git #### Rationale This was done because to avoid [thread deadlocks](https://github.com/ruby/net-imap/issues/14), `MailRoom` needs -an updated version of the `net-imap` gem. However, this [version of the net-imap cannot be installed by an unprivileged -user](https://github.com/ruby/net-imap/issues/14) due to [an error installing the digest -gem](https://github.com/ruby/digest/issues/14). [This bug in the Ruby interpreter](https://bugs.ruby-lang.org/issues/17761) was fixed in Ruby +an updated version of the `net-imap` gem. However, this +[version of the net-imap cannot be installed by an unprivileged user](https://github.com/ruby/net-imap/issues/14) due to +[an error installing the digest gem](https://github.com/ruby/digest/issues/14). +[This bug in the Ruby interpreter](https://bugs.ruby-lang.org/issues/17761) was fixed in Ruby 3.0.2. Updating the gem directly in the GitLab Rails `Gemfile` caused a [production incident](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4053) diff --git a/doc/development/event_store.md b/doc/development/event_store.md index ffde51216cf..37035083e23 100644 --- a/doc/development/event_store.md +++ b/doc/development/event_store.md @@ -223,6 +223,15 @@ Gitlab::EventStore.publish( ) ``` +Events should be dispatched from the relevant Service class whenever possible. Some +exceptions exist where we may allow models to publish events, like in state machine transitions. +For example, instead of scheduling `Ci::BuildFinishedWorker`, which runs a collection of side effects, +we could publish a `Ci::BuildFinishedEvent` and let other domains react asynchronously. + +`ActiveRecord` callbacks are too low-level to represent a domain event. They represent more database +record changes. There might be cases where it would make sense, but we should consider +those exceptions. + ## Create a subscriber A subscriber is a Sidekiq worker that includes the `Gitlab::EventStore::Subscriber` module. @@ -320,7 +329,7 @@ it 'publishes a ProjectCreatedEvent with project id and namespace id' do # The project ID will only be generated when the `create_project` # is called in the expect block. expected_data = { project_id: kind_of(Numeric), namespace_id: group_id } - + expect { create_project(user, name: 'Project', path: 'project', namespace_id: group_id) } .to publish_event(Projects::ProjectCreatedEvent) .with(expected_data) diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index 2a1083d031f..bdd6c5d6e84 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -13,7 +13,7 @@ This page contains guidelines we should follow. ## Quick summary -Since [no ARIA is better than bad ARIA](https://www.w3.org/TR/wai-aria-practices/#no_aria_better_bad_aria), +Since [no ARIA is better than bad ARIA](https://w3c.github.io/aria-practices/#no_aria_better_bad_aria), review the following recommendations before using `aria-*`, `role`, and `tabindex`. Use semantic HTML, which has accessibility semantics baked in, and ideally test with [relevant combinations of screen readers and browsers](https://www.accessibility-developer-guide.com/knowledge/screen-readers/relevant-combinations/). diff --git a/doc/development/fe_guide/architecture.md b/doc/development/fe_guide/architecture.md index afaf6df8f8a..1d08296eafc 100644 --- a/doc/development/fe_guide/architecture.md +++ b/doc/development/fe_guide/architecture.md @@ -11,7 +11,7 @@ When developing a feature that requires architectural design, or changing the fu A Frontend Architect is an expert who makes high-level Frontend design decisions and decides on technical standards, including coding standards and frameworks. -Architectural decisions should be accessible to everyone, so please document +Architectural decisions should be accessible to everyone, so document them in the relevant Merge Request discussion or by updating our documentation when appropriate. @@ -19,7 +19,7 @@ You can find the Frontend Architecture experts on the [team page](https://about. ## Widget Architecture -The [Plan stage](https://about.gitlab.com/handbook/engineering/development/dev/fe-plan/) +The [Plan stage](https://about.gitlab.com/handbook/engineering/development/dev/plan-project-management/) is refactoring the right sidebar to consist of **widgets**. They have a specific architecture to be reusable and to expose an interface that can be used by external Vue applications on the page. Learn more about the [widget architecture](widgets.md). diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index d4c29cb8a24..f262e48b6da 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -296,6 +296,7 @@ const builtInContentEditorExtensions = [ Dropcursor, Emoji, // Other extensions +] ``` ### The Markdown serializer diff --git a/doc/development/fe_guide/design_anti_patterns.md b/doc/development/fe_guide/design_anti_patterns.md index b7238bb2813..580f488bd33 100644 --- a/doc/development/fe_guide/design_anti_patterns.md +++ b/doc/development/fe_guide/design_anti_patterns.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Anti-patterns may seem like good approaches at first, but it has been shown that they bring more ills than benefits. These should generally be avoided. -Throughout the GitLab codebase, there may be historic uses of these anti-patterns. Please [use discretion](https://about.gitlab.com/handbook/engineering/development/principles/#balance-refactoring-and-velocity) +Throughout the GitLab codebase, there may be historic uses of these anti-patterns. [Use discretion](https://about.gitlab.com/handbook/engineering/development/principles/#balance-refactoring-and-velocity) when figuring out whether or not to refactor, when touching code that uses one of these legacy patterns. NOTE: @@ -62,7 +62,7 @@ could be appropriate: - When a responsibility is truly global and should be referenced across the application (for example, an application-wide Event Bus). -Even in these scenarios, please consider avoiding the Shared Global Object pattern because the +Even in these scenarios, consider avoiding the Shared Global Object pattern because the side-effects can be notoriously difficult to reason with. ### References @@ -140,7 +140,7 @@ that a Singleton could be appropriate in the following rare cases: - We need to manage some resource that **MUST** have just 1 instance (that is, some hardware restriction). - There is a real [cross-cutting concern](https://en.wikipedia.org/wiki/Cross-cutting_concern) (for example, logging) and a Singleton provides the simplest API. -Even in these scenarios, please consider avoiding the Singleton pattern. +Even in these scenarios, consider avoiding the Singleton pattern. ### What alternatives are there to the Singleton pattern? diff --git a/doc/development/fe_guide/development_process.md b/doc/development/fe_guide/development_process.md index b4893fd4ef9..3273263de3b 100644 --- a/doc/development/fe_guide/development_process.md +++ b/doc/development/fe_guide/development_process.md @@ -16,7 +16,7 @@ Copy the content over to your issue or merge request and if something doesn't ap This checklist is intended to help us during development of bigger features/refactorings. It is not a "use it always and every point always matches" list. -Please use your best judgment when to use it and please contribute new points through merge requests if something comes to your mind. +Use your best judgment when to use it and contribute new points through merge requests if something comes to your mind. ```markdown ### Frontend development @@ -39,7 +39,7 @@ Please use your best judgment when to use it and please contribute new points th - [ ] **Cookie Mode** Think about hiding the feature behind a cookie flag if the implementation is on top of existing features - [ ] **New route** Are you refactoring something big then you might consider adding a new route where you implement the new feature and when finished delete the current route and rename the new one. (for example 'merge_request' and 'new_merge_request') - [ ] **Setup** Is there any specific setup needed for your implementation (for example a kubernetes cluster)? Then let everyone know if it is not already mentioned where they can find documentation (if it doesn't exist - create it) -- [ ] **Security** Are there any new security relevant implementations? Then please contact the security team for an app security review. If you are not sure ask our [domain expert](https://about.gitlab.com/handbook/engineering/frontend/#frontend-domain-experts) +- [ ] **Security** Are there any new security relevant implementations? Then contact the security team for an app security review. If you are not sure ask our [domain expert](https://about.gitlab.com/handbook/engineering/frontend/#frontend-domain-experts) #### During development @@ -90,7 +90,7 @@ code that is unused: ### Merge Request Review -With the purpose of being [respectful of others' time](https://about.gitlab.com/handbook/values/#be-respectful-of-others-time) please follow these guidelines when asking for a review: +With the purpose of being [respectful of others' time](https://about.gitlab.com/handbook/values/#be-respectful-of-others-time), follow these guidelines when asking for a review: - Make sure your Merge Request: - milestone is set @@ -101,7 +101,7 @@ With the purpose of being [respectful of others' time](https://about.gitlab.com/ - includes tests - includes a changelog entry (when necessary) - Before assigning to a maintainer, assign to a reviewer. -- If you assigned a merge request or pinged someone directly, be patient because we work in different timezones and asynchronously. Unless the merge request is urgent (like fixing a broken default branch), please don't DM or reassign the merge request before waiting for a 24-hour window. +- If you assigned a merge request or pinged someone directly, be patient because we work in different timezones and asynchronously. Unless the merge request is urgent (like fixing a broken default branch), don't DM or reassign the merge request before waiting for a 24-hour window. - If you have a question regarding your merge request/issue, make it on the merge request/issue. When we DM each other, we no longer have a SSOT and [no one else is able to contribute](https://about.gitlab.com/handbook/values/#public-by-default). - When you have a big **Draft** merge request with many changes, you're advised to get the review started before adding/removing significant code. Make sure it is assigned well before the release cut-off, as the reviewers/maintainers would always prioritize reviewing finished MRs before the **Draft** ones. - Make sure to remove the `Draft:` title before the last round of review. diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index 39c39894dac..6a645416c0a 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Rules of Frontend FAQ 1. **You talk about Frontend FAQ.** - Please share links to it whenever applicable, so more eyes catch when content + Share links to it whenever applicable, so more eyes catch when content gets outdated. 1. **Keep it short and simple.** Whenever an answer needs more than two sentences it does not belong here. @@ -17,7 +17,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Linking to relevant source code, issue / epic, or other documentation helps to understand the answer. 1. **If you see something, do something.** - Please remove or update any content that is outdated as soon as you see it. + Remove or update any content that is outdated as soon as you see it. ## FAQ @@ -101,7 +101,7 @@ axios.get(joinPaths(gon.gitlab_url, '-', 'foo')) axios.get(joinPaths(gon.relative_url_root, '-', 'foo')) ``` -Also, please try not to hardcode paths in the Frontend, but instead receive them from the Backend (see next section). +Also, try not to hardcode paths in the Frontend, but instead receive them from the Backend (see next section). When referencing Backend rails paths, avoid using `*_url`, and use `*_path` instead. Example: diff --git a/doc/development/fe_guide/graphql.md b/doc/development/fe_guide/graphql.md index 10db332d64c..442dda20d23 100644 --- a/doc/development/fe_guide/graphql.md +++ b/doc/development/fe_guide/graphql.md @@ -14,7 +14,7 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo **General resources**: - [📚 Official Introduction to GraphQL](https://graphql.org/learn/) -- [📚 Official Introduction to Apollo](https://www.apollographql.com/docs/tutorial/introduction/) +- [📚 Official Introduction to Apollo](https://www.apollographql.com/tutorials/fullstack-quickstart/introduction) **GraphQL at GitLab**: @@ -109,7 +109,7 @@ Default client accepts two parameters: `resolvers` and `config`. If you are making multiple queries to the same Apollo client object you might encounter the following error: `Cache data may be lost when replacing the someProperty field of a Query object. To address this problem, either ensure all objects of SomeEntityhave an id or a custom merge function`. We are already checking `ID` presence for every GraphQL type that has an `ID`, so this shouldn't be the case. Most likely, the `SomeEntity` type doesn't have an `ID` property, and to fix this warning we need to define a custom merge function. -We have some client-wide types with `merge: true` defined in the default client as [typePolicies](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/lib/graphql.js) (this means that Apollo will merge existing and incoming responses in the case of subsequent queries). Please consider adding `SomeEntity` there or defining a custom merge function for it. +We have some client-wide types with `merge: true` defined in the default client as [typePolicies](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/lib/graphql.js) (this means that Apollo will merge existing and incoming responses in the case of subsequent queries). Consider adding `SomeEntity` there or defining a custom merge function for it. ## GraphQL Queries @@ -212,7 +212,7 @@ with a **new and updated** object. To facilitate the process of updating the cache and returning the new object we use the library [Immer](https://immerjs.github.io/immer/). -Please, follow these conventions: +Follow these conventions: - The updated cache is named `data`. - The original cache data is named `sourceData`. @@ -597,7 +597,7 @@ export default { Note that, even if the directive evaluates to `false`, the guarded entity is sent to the backend and matched against the GraphQL schema. So this approach requires that the feature-flagged entity exists in the schema, even if the feature flag is disabled. When the feature flag is turned off, it -is recommended that the resolver returns `null` at the very least using the same feature flag as the frontend. See the [API GraphQL guide](../api_graphql_styleguide.md#frontend-and-backend-feature-flag-strategies). +is recommended that the resolver returns `null` at the very least using the same feature flag as the frontend. See the [API GraphQL guide](../api_graphql_styleguide.md#feature-flags). ##### Different versions of a query @@ -729,8 +729,9 @@ In this case, we can either: - Skip passing a cursor. - Pass `null` explicitly to `after`. -After data is fetched, we can use the `update`-hook as an opportunity [to customize -the data that is set in the Vue component property](https://apollo.vuejs.org/api/smart-query.html#options). This allows us to get a hold of the `pageInfo` object among other data. +After data is fetched, we can use the `update`-hook as an opportunity +[to customize the data that is set in the Vue component property](https://apollo.vuejs.org/api/smart-query.html#options). +This allows us to get a hold of the `pageInfo` object among other data. In the `result`-hook, we can inspect the `pageInfo` object to see if we need to fetch the next page. Note that we also keep a `requestCount` to ensure that the application @@ -895,6 +896,51 @@ export default new VueApollo({ This is similar to the `DesignCollection` example above as new page results are appended to the previous ones. +For some cases, it's hard to define the correct `keyArgs` for the field because all +the fields are updated. In this case, we can set `keyArgs` to `false`. This instructs +Apollo Client to not perform any automatic merge, and fully rely on the logic we +put into the `merge` function. + +For example, we have a query like this: + +```javascript +query searchGroupsWhereUserCanTransfer { + currentUser { + id + groups { + nodes { + id + fullName + } + pageInfo { + ...PageInfo + } + } + } +} +``` + +Here, the `groups` field doesn't have a good candidate for `keyArgs`: both +`nodes` and `pageInfo` will be updated when we're fetching a second page. +Setting `keyArgs` to `false` makes the update work as intended: + +```javascript +typePolicies: { + UserCore: { + fields: { + groups: { + keyArgs: false, + }, + }, + }, + GroupConnection: { + fields: { + nodes: concatPagination(), + }, + }, +} +``` + #### Using a recursive query in components When it is necessary to fetch all paginated data initially an Apollo query can do the trick for us. @@ -1444,7 +1490,7 @@ describe('Some component', () => { When mocking resolved values, ensure the structure of the response is the same as the actual API response. For example, root property should be `data`. -When testing queries, please keep in mind they are promises, so they need to be _resolved_ to render a result. Without resolving, we can check the `loading` state of the query: +When testing queries, keep in mind they are promises, so they need to be _resolved_ to render a result. Without resolving, we can check the `loading` state of the query: ```javascript it('renders a loading state', () => { @@ -2001,11 +2047,15 @@ relative to `app/graphql/queries` folder: for example, if we need a ### Mocked client returns empty objects instead of mock response -If your unit test is failing because response contains empty objects instead of mock data, you would need to add `__typename` field to the mocked response. This happens because mocked client (unlike the real one) does not populate the response with typenames and in some cases we need to do it manually so the client is able to recognize a GraphQL type. +If your unit test is failing because the response contains empty objects instead of mock data, add +`__typename` field to the mocked responses. + +Alternatively, [GraphQL query fixtures](../testing_guide/frontend_testing.md#graphql-query-fixtures) +automatically adds the `__typename` for you upon generation. ### Warning about losing cache data -Sometimes you can see a warning in the console: `Cache data may be lost when replacing the someProperty field of a Query object. To address this problem, either ensure all objects of SomeEntityhave an id or a custom merge function`. Please check section about [multiple queries](#multiple-client-queries-for-the-same-object) to resolve an issue. +Sometimes you can see a warning in the console: `Cache data may be lost when replacing the someProperty field of a Query object. To address this problem, either ensure all objects of SomeEntityhave an id or a custom merge function`. Check section about [multiple queries](#multiple-client-queries-for-the-same-object) to resolve an issue. ```yaml - current_route_path = request.fullpath.match(/-\/tree\/[^\/]+\/(.+$)/).to_a[1] diff --git a/doc/development/fe_guide/icons.md b/doc/development/fe_guide/icons.md index d107af156db..73f196ef51f 100644 --- a/doc/development/fe_guide/icons.md +++ b/doc/development/fe_guide/icons.md @@ -81,7 +81,7 @@ export default { ### Usage in HTML/JS -Please use the following function inside JS to render an icon: +Use the following function inside JS to render an icon: `gl.utils.spriteIcon(iconName)` ## Loading icon diff --git a/doc/development/fe_guide/index.md b/doc/development/fe_guide/index.md index 544985d7edc..02086ec5f1b 100644 --- a/doc/development/fe_guide/index.md +++ b/doc/development/fe_guide/index.md @@ -147,7 +147,7 @@ Best practices for [client-side logging](logging.md) for GitLab frontend develop ## [Internationalization (i18n) and Translations](../i18n/externalization.md) -Frontend internationalization support is described in [this document](../i18n/). +Frontend internationalization support is described in [this document](../i18n/index.md). The [externalization part of the guide](../i18n/externalization.md) explains the helpers/methods available. ## [Troubleshooting](troubleshooting.md) diff --git a/doc/development/fe_guide/merge_request_widget_extensions.md b/doc/development/fe_guide/merge_request_widget_extensions.md new file mode 100644 index 00000000000..a2ff10cc57f --- /dev/null +++ b/doc/development/fe_guide/merge_request_widget_extensions.md @@ -0,0 +1,437 @@ +--- +stage: Create +group: Code Review +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Merge request widget extensions **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44616) in GitLab 13.6. + +Extensions in the merge request widget enable you to add new features +into the merge request widget that match the design framework. +With extensions we get a lot of benefits out of the box without much effort required, like: + +- A consistent look and feel. +- Tracking when the extension is opened. +- Virtual scrolling for performance. + +## Usage + +To use extensions you must first create a new extension object to fetch the +data to render in the extension. For a working example, refer to the example file in +`app/assets/javascripts/vue_merge_request_widget/extensions/issues.js`. + +The basic object structure: + +```javascript +export default { + name: '', // Required: This helps identify the widget + props: [], // Required: Props passed from the widget state + i18n: { // Required: Object to hold i18n text + label: '', // Required: Used for tooltips and aria-labels + loading: '', // Required: Loading text for when data is loading + }, + expandEvent: '', // Optional: RedisHLL event name to track expanding content + enablePolling: false, // Optional: Tells extension to poll for data + modalComponent: null, // Optional: The component to use for the modal + telemetry: true, // Optional: Reports basic telemetry for the extension. Set to false to disable telemetry + computed: { + summary(data) {}, // Required: Level 1 summary text + statusIcon(data) {}, // Required: Level 1 status icon + tertiaryButtons() {}, // Optional: Level 1 action buttons + shouldCollapse() {}, // Optional: Add logic to determine if the widget can expand or not + }, + methods: { + fetchCollapsedData(props) {}, // Required: Fetches data required for collapsed state + fetchFullData(props) {}, // Required: Fetches data for the full expanded content + fetchMultiData() {}, // Optional: Works in conjunction with `enablePolling` and allows polling multiple endpoints + }, +}; +``` + +By following the same data structure, each extension can follow the same registering structure, +but each extension can manage its data sources. + +After creating this structure, you must register it. You can register the extension at any +point _after_ the widget has been created. To register a extension: + +```javascript +// Import the register method +import { registerExtension } from '~/vue_merge_request_widget/components/extensions'; + +// Import the new extension +import issueExtension from '~/vue_merge_request_widget/extensions/issues'; + +// Register the imported extension +registerExtension(issueExtension); +``` + +## Data fetching + +Each extension must fetch data. Fetching is handled when registering the extension, +not by the core component itself. This approach allows for various different +data fetching methods to be used, such as GraphQL or REST API calls. + +### API calls + +For performance reasons, it is best if the collapsed state fetches only the data required to +render the collapsed state. This fetching happens in the `fetchCollapsedData` method. +This method is called with the props as an argument, so you can easily access +any paths set in the state. + +To allow the extension to set the data, this method **must** return the data. No +special formatting is required. When the extension receives this data, +it is set to `collapsedData`. You can access `collapsedData` in any computed property or +method. + +When the user clicks **Expand**, the `fetchFullData` method is called. This method +also gets called with the props as an argument. This method **must** also return +the full data. However, this data must be correctly formatted to match the format +mentioned in the data structure section. + +#### Technical debt + +For some of the current extensions, there is no split in data fetching. All the data +is fetched through the `fetchCollapsedData` method. While less performant, +it allows for faster iteration. + +To handle this the `fetchFullData` returns the data set through +the `fetchCollapsedData` method call. In these cases, the `fetchFullData` must +return a promise: + +```javascript +fetchCollapsedData() { + return ['Some data']; +}, +fetchFullData() { + return Promise.resolve(this.collapsedData) +}, +``` + +### Data structure + +The data returned from `fetchFullData` must match the format below. This format +allows the core component to render the data in a way that matches +the design framework. Any text properties can use the styling placeholders +mentioned below: + +```javascript +{ + id: data.id, // Required: ID used as a key for each row + header: 'Header' || ['Header', 'sub-header'], // Required: String or array can be used for the header text + text: '', // Required: Main text for the row + subtext: '', // Optional: Smaller sub-text to be displayed below the main text + icon: { // Optional: Icon object + name: EXTENSION_ICONS.success, // Required: The icon name for the row + }, + badge: { // Optional: Badge displayed after text + text: '', // Required: Text to be displayed inside badge + variant: '', // Optional: GitLab UI badge variant, defaults to info + }, + link: { // Optional: Link to a URL displayed after text + text: '', // Required: Text of the link + href: '', // Optional: URL for the link + }, + modal: { // Optional: Link to open a modal displayed after text + text: '', // Required: Text of the link + onClick: () => {} // Optional: Function to run when link is clicked, i.e. to set this.modalData + } + actions: [], // Optional: Action button for row + children: [], // Optional: Child content to render, structure matches the same structure +} +``` + +### Polling + +To enable polling for an extension, an options flag must be present in the extension: + +```javascript +export default { + //... + enablePolling: true +}; +``` + +This flag tells the base component we should poll the `fetchCollapsedData()` +defined in the extension. Polling stops if the response has data, or if an error is present. + +When writing the logic for `fetchCollapsedData()`, a complete Axios response must be returned +from the method. The polling utility needs data like polling headers to work correctly: + +```javascript +export default { + //... + enablePolling: true + methods: { + fetchCollapsedData() { + return axios.get(this.reportPath) + }, + }, +}; +``` + +Most of the time the data returned from the extension's endpoint is not in the format +the UI needs. We must format the data before setting the collapsed data in the base component. + +If the computed property `summary` can rely on `collapsedData`, you can format the data +when `fetchFullData` is invoked: + +```javascript +export default { + //... + enablePolling: true + methods: { + fetchCollapsedData() { + return axios.get(this.reportPath) + }, + fetchFullData() { + return Promise.resolve(this.prepareReports()); + }, + // custom method + prepareReports() { + // unpack values from collapsedData + const { new_errors, existing_errors, resolved_errors } = this.collapsedData; + + // perform data formatting + + return [...newErrors, ...existingErrors, ...resolvedErrors] + } + }, +}; +``` + +If the extension relies on `collapsedData` being formatted before invoking `fetchFullData()`, +then `fetchCollapsedData()` must return the Axios response as well as the formatted data: + +```javascript +export default { + //... + enablePolling: true + methods: { + fetchCollapsedData() { + return axios.get(this.reportPath).then(res => { + const formattedData = this.prepareReports(res.data) + + return { + ...res, + data: formattedData, + } + }) + }, + // Custom method + prepareReports() { + // Unpack values from collapsedData + const { new_errors, existing_errors, resolved_errors } = this.collapsedData; + + // Perform data formatting + + return [...newErrors, ...existingErrors, ...resolvedErrors] + } + }, +}; +``` + +If the extension must poll multiple endpoints at the same time, then `fetchMultiData` +can be used to return an array of functions. A new `poll` object is created for each +endpoint and they are polled separately. After all endpoints are resolved, polling is +stopped and `setCollapsedData` is called with an array of `response.data`. + +```javascript +export default { + //... + enablePolling: true + methods: { + fetchMultiData() { + return [ + () => axios.get(this.reportPath1), + () => axios.get(this.reportPath2), + () => axios.get(this.reportPath3) + }, + }, +}; +``` + +WARNING: +The function must return a `Promise` that resolves the `response` object. +The implementation relies on the `POLL-INTERVAL` header to keep polling, therefore it is +important not to alter the status code and headers. + +### Errors + +If `fetchCollapsedData()` or `fetchFullData()` methods throw an error: + +- The loading state of the extension is updated to `LOADING_STATES.collapsedError` + and `LOADING_STATES.expandedError` respectively. +- The extensions header displays an error icon and updates the text to be either: + - The text defined in `$options.i18n.error`. + - "Failed to load" if `$options.i18n.error` is not defined. +- The error is sent to Sentry to log that it occurred. + +To customise the error text, add it to the `i18n` object in your extension: + +```javascript +export default { + //... + i18n: { + //... + error: __('Your error text'), + }, +}; +``` + +## Telemetry + +The base implementation of the widget extension framework includes some telemetry events. +Each widget reports: + +- `view`: When it is rendered to the screen. +- `expand`: When it is expanded. +- `full_report_clicked`: When an (optional) input is clicked to view the full report. +- Outcome (`expand_success`, `expand_warning`, or `expand_failed`): One of three + additional events relating to the status of the widget when it was expanded. + +### Add new widgets + +When adding new widgets, the above events must be marked as `known`, and have metrics +created, to be reportable. + +NOTE: +Events that are only for EE should include `--ee` at the end of both shell commands below. + +To generate these known events for a single widget: + +1. Widgets should be named `Widget${CamelName}`. + - For example: a widget for **Test Reports** should be `WidgetTestReports`. +1. Compute the widget name slug by converting the `${CamelName}` to lower-, snake-case. + - The previous example would be `test_reports`. +1. Add the new widget name slug to `lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb` + in the `WIDGETS` list. +1. Ensure the GDK is running (`gdk start`). +1. Generate known events on the command line with the following command. + Replace `test_reports` with your appropriate name slug: + + ```shell + bundle exec rails generate gitlab:usage_metric_definition \ + counts.i_code_review_merge_request_widget_test_reports_count_view \ + counts.i_code_review_merge_request_widget_test_reports_count_full_report_clicked \ + counts.i_code_review_merge_request_widget_test_reports_count_expand \ + counts.i_code_review_merge_request_widget_test_reports_count_expand_success \ + counts.i_code_review_merge_request_widget_test_reports_count_expand_warning \ + counts.i_code_review_merge_request_widget_test_reports_count_expand_failed \ + --dir=all + ``` + +1. Modify each newly generated file to match the existing files for the merge request widget extension telemetry. + - Find existing examples by doing a glob search, like: `metrics/**/*_i_code_review_merge_request_widget_*` + - Roughly speaking, each file should have these values: + 1. `description` = A plain English description of this value. Review existing widget extension telemetry files for examples. + 1. `product_section` = `dev` + 1. `product_stage` = `create` + 1. `product_group` = `code_review` + 1. `product_category` = `code_review` + 1. `introduced_by_url` = `'[your MR]'` + 1. `options.events` = (the event in the command from above that generated this file, like `i_code_review_merge_request_widget_test_reports_count_view`) + - This value is how the telemetry events are linked to "metrics" so this is probably one of the more important values. + 1. `data_source` = `redis` + 1. `data_category` = `optional` +1. Generate known HLL events on the command line with the following command. + Replace `test_reports` with your appropriate name slug. + + ```shell + bundle exec rails generate gitlab:usage_metric_definition:redis_hll code_review \ + i_code_review_merge_request_widget_test_reports_view \ + i_code_review_merge_request_widget_test_reports_full_report_clicked \ + i_code_review_merge_request_widget_test_reports_expand \ + i_code_review_merge_request_widget_test_reports_expand_success \ + i_code_review_merge_request_widget_test_reports_expand_warning \ + i_code_review_merge_request_widget_test_reports_expand_failed \ + --class_name=RedisHLLMetric + ``` + +1. Repeat step 6, but change the `data_source` to `redis_hll`. +1. Add each of the HLL metrics to `lib/gitlab/usage_data_counters/known_events/code_review_events.yml`: + 1. `name` = (the event) + 1. `redis_slot` = `code_review` + 1. `category` = `code_review` + 1. `aggregation` = `weekly` +1. Add each event to the appropriate aggregates in `config/metrics/aggregates/code_review.yml` + +### Add new events + +If you are adding a new event to our known events, include the new event in the +`KNOWN_EVENTS` list in `lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb`. + +## Icons + +Level 1 and all subsequent levels can have their own status icons. To keep with +the design framework, import the `EXTENSION_ICONS` constant +from the `constants.js` file: + +```javascript +import { EXTENSION_ICONS } from '~/vue_merge_request_widget/constants.js'; +``` + +This constant has the below icons available for use. Per the design framework, +only some of these icons should be used on level 1: + +- `failed` +- `warning` +- `success` +- `neutral` +- `error` +- `notice` +- `severityCritical` +- `severityHigh` +- `severityMedium` +- `severityLow` +- `severityInfo` +- `severityUnknown` + +## Text styling + +Any area that has text can be styled with the placeholders below. This +technique follows the same technique as `sprintf`. However, instead of specifying +these through `sprintf`, the extension does this automatically. + +Every placeholder contains starting and ending tags. For example, `success` uses +`Hello %{success_start}world%{success_end}`. The extension then +adds the start and end tags with the correct styling classes. + +| Placeholder | Style | +|-------------|-----------------------------------------| +| success | `gl-font-weight-bold gl-text-green-500` | +| danger | `gl-font-weight-bold gl-text-red-500` | +| critical | `gl-font-weight-bold gl-text-red-800` | +| same | `gl-font-weight-bold gl-text-gray-700` | +| strong | `gl-font-weight-bold` | +| small | `gl-font-sm` | + +## Action buttons + +You can add action buttons to all level 1 and 2 in each extension. These buttons +are meant as a way to provide links or actions for each row: + +- Action buttons for level 1 can be set through the `tertiaryButtons` computed property. + This property should return an array of objects for each action button. +- Action buttons for level 2 can be set by adding the `actions` key to the level 2 rows object. + The value for this key must also be an array of objects for each action button. + +Links must follow this structure: + +```javascript +{ + text: 'Click me', + href: this.someLinkHref, + target: '_blank', // Optional +} +``` + +For internal action buttons, follow this structure: + +```javascript +{ + text: 'Click me', + onClick() {} +} +``` diff --git a/doc/development/fe_guide/performance.md b/doc/development/fe_guide/performance.md index bcdc49a1070..2e1fabd739c 100644 --- a/doc/development/fe_guide/performance.md +++ b/doc/development/fe_guide/performance.md @@ -8,6 +8,21 @@ info: To determine the technical writer assigned to the Stage/Group associated w Performance is an essential part and one of the main areas of concern for any modern application. +## Monitoring + +We have a performance dashboard available in one of our [Grafana instances](https://dashboards.gitlab.net/d/000000043/sitespeed-page-summary?orgId=1). This dashboard automatically aggregates metric data from [sitespeed.io](https://www.sitespeed.io/) every 4 hours. These changes are displayed after a set number of pages are aggregated. + +These pages can be found inside text files in the [`sitespeed-measurement-setup` repository](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup) called [`gitlab`](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup/-/tree/master/gitlab) +Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing URLs of pages to the text files. The changes are pushed live on the next scheduled run after the changes are merged into `main`. + +There are 3 recommended high impact metrics (core web vitals) to review on each page: + +- [Largest Contentful Paint](https://web.dev/lcp/) +- [First Input Delay](https://web.dev/fid/) +- [Cumulative Layout Shift](https://web.dev/cls/) + +For these metrics, lower numbers are better as it means that the website is more performant. + ## User Timing API [User Timing API](https://developer.mozilla.org/en-US/docs/Web/API/User_Timing_API) is a web API @@ -77,9 +92,9 @@ performance.getEntriesByType('mark'); performance.getEntriesByType('measure'); ``` -Using `getEntriesByName()` or `getEntriesByType()` returns an Array of [the PerformanceMeasure -objects](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceMeasure) which contain -information about the measurement's start time and duration. +Using `getEntriesByName()` or `getEntriesByType()` returns an Array of +[the PerformanceMeasure objects](https://developer.mozilla.org/en-US/docs/Web/API/PerformanceMeasure) +which contain information about the measurement's start time and duration. ### User Timing API utility @@ -220,7 +235,7 @@ Use the following rules when creating real-time solutions. A `Poll-Interval: -1` means you should disable polling, and this must be implemented. 1. A response with HTTP status different from 2XX should disable polling as well. 1. Use a common library for polling. -1. Poll on active tabs only. Please use [Visibility](https://github.com/ai/visibilityjs). +1. Poll on active tabs only. Use [Visibility](https://github.com/ai/visibilityjs). 1. Use regular polling intervals, do not use backoff polling or jitter, as the interval is controlled by the server. 1. The backend code is likely to be using ETags. You do not and should not check for status @@ -434,7 +449,7 @@ Use `webpackChunkName` when generating dynamic imports as it provides a deterministic filename for the chunk which can then be cached in the browser across GitLab versions. -More information is available in [webpack's code splitting documentation](https://webpack.js.org/guides/code-splitting/#dynamic-imports) and [vue's dynamic component documentation](https://vuejs.org/v2/guide/components-dynamic-async.html). +More information is available in [webpack's code splitting documentation](https://webpack.js.org/guides/code-splitting/#dynamic-imports) and [vue's dynamic component documentation](https://v2.vuejs.org/v2/guide/components-dynamic-async.html). ### Minimizing page size diff --git a/doc/development/fe_guide/source_editor.md b/doc/development/fe_guide/source_editor.md index b06e341630f..88508e94380 100644 --- a/doc/development/fe_guide/source_editor.md +++ b/doc/development/fe_guide/source_editor.md @@ -129,7 +129,7 @@ with additional functions on the instance level: Source Editor provides a universal, extensible editing tool to the whole product, and doesn't depend on any particular group. Even though the Source Editor's core is owned by -[Create::Editor FE Team](https://about.gitlab.com/handbook/engineering/development/dev/create-editor/), +[Create::Editor FE Team](https://about.gitlab.com/handbook/engineering/development/dev/create/editor/), any group can own the extensions—the main functional elements. The goal of Source Editor extensions is to keep the editor's core slim and stable. Any needed features can be added as extensions to this core. Any group can diff --git a/doc/development/fe_guide/storybook.md b/doc/development/fe_guide/storybook.md index 4c0e7b2612b..45342eb6d72 100644 --- a/doc/development/fe_guide/storybook.md +++ b/doc/development/fe_guide/storybook.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Storybook -The Storybook for the `gitlab-org/gitlab` project is available on our [GitLab Pages site](https://gitlab-org.gitlab.io/gitlab/storybook). +The Storybook for the `gitlab-org/gitlab` project is available on our [GitLab Pages site](https://gitlab-org.gitlab.io/gitlab/storybook/). ## Storybook in local development diff --git a/doc/development/fe_guide/style/javascript.md b/doc/development/fe_guide/style/javascript.md index d93dc8292d4..b86bdfafa21 100644 --- a/doc/development/fe_guide/style/javascript.md +++ b/doc/development/fe_guide/style/javascript.md @@ -123,7 +123,8 @@ things.map(parseInt); things.map(Number); ``` -**PLEASE NOTE:** If the String could represent a non-integer (i.e., it includes a decimal), **do not** use `parseInt`. Consider `Number` or `parseFloat` instead. +NOTE: +If the String could represent a non-integer (i.e., it includes a decimal), **do not** use `parseInt`. Consider `Number` or `parseFloat` instead. ## CSS Selectors - Use `js-` prefix diff --git a/doc/development/fe_guide/style/scss.md b/doc/development/fe_guide/style/scss.md index 451b0c8a4c6..17e80762a38 100644 --- a/doc/development/fe_guide/style/scss.md +++ b/doc/development/fe_guide/style/scss.md @@ -12,7 +12,7 @@ easy to maintain, and performant for the end-user. ## Rules -Our CSS is a mixture of current and legacy approaches. That means sometimes it may be difficult to follow this guide to the letter; it means you are likely to run into exceptions, where following the guide is difficult to impossible without major effort. In those cases, you may work with your reviewers and maintainers to identify an approach that does not fit these rules. Please endeavor to limit these cases. +Our CSS is a mixture of current and legacy approaches. That means sometimes it may be difficult to follow this guide to the letter; it means you are likely to run into exceptions, where following the guide is difficult to impossible without major effort. In those cases, you may work with your reviewers and maintainers to identify an approach that does not fit these rules. Try to limit these cases. ### Utility Classes diff --git a/doc/development/fe_guide/style/vue.md b/doc/development/fe_guide/style/vue.md index 5c79d47e7b0..c9bd0e1b35a 100644 --- a/doc/development/fe_guide/style/vue.md +++ b/doc/development/fe_guide/style/vue.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Linting We default to [eslint-vue-plugin](https://github.com/vuejs/eslint-plugin-vue), with the `plugin:vue/recommended`. -Please check this [rules](https://github.com/vuejs/eslint-plugin-vue#bulb-rules) for more documentation. +Check the [rules](https://github.com/vuejs/eslint-plugin-vue#bulb-rules) for more documentation. ## Basic Rules @@ -448,9 +448,9 @@ Typically, when testing a Vue component, the component should be "re-mounted" in To achieve this: 1. Create a mutable `wrapper` variable inside the top-level `describe` block. -1. Mount the component using [`mount`](https://vue-test-utils.vuejs.org/api/#mount)/ -[`shallowMount`](https://vue-test-utils.vuejs.org/api/#shallowMount). -1. Reassign the resulting [`Wrapper`](https://vue-test-utils.vuejs.org/api/wrapper/#wrapper) +1. Mount the component using [`mount`](https://v1.test-utils.vuejs.org/api/#mount)/ +[`shallowMount`](https://v1.test-utils.vuejs.org/api/#shallowMount). +1. Reassign the resulting [`Wrapper`](https://v1.test-utils.vuejs.org/api/wrapper/#wrapper) instance to our `wrapper` variable. Creating a global, mutable wrapper provides a number of advantages, including the ability to: @@ -476,8 +476,8 @@ Creating a global, mutable wrapper provides a number of advantages, including th To avoid duplicating our mounting logic, it's useful to define a `createComponent` factory function that we can reuse in each test block. This is a closure which should reassign our `wrapper` variable -to the result of [`mount`](https://vue-test-utils.vuejs.org/api/#mount) and -[`shallowMount`](https://vue-test-utils.vuejs.org/api/#shallowMount): +to the result of [`mount`](https://v1.test-utils.vuejs.org/api/#mount) and +[`shallowMount`](https://v1.test-utils.vuejs.org/api/#shallowMount): ```javascript import MyComponent from '~/path/to/my_component.vue'; @@ -579,9 +579,9 @@ the mounting function (`mount` or `shallowMount`) to be used to mount the compon ### Setting component state -1. Avoid using [`setProps`](https://vue-test-utils.vuejs.org/api/wrapper/#setprops) to set +1. Avoid using [`setProps`](https://v1.test-utils.vuejs.org/api/wrapper/#setprops) to set component state wherever possible. Instead, set the component's -[`propsData`](https://vue-test-utils.vuejs.org/api/options.html#propsdata) when mounting the component: +[`propsData`](https://v1.test-utils.vuejs.org/api/options.html#propsdata) when mounting the component: ```javascript // bad @@ -659,7 +659,7 @@ The goal of this accord is to make sure we are all on the same page. 1. If an outside jQuery Event needs to be listen to inside the Vue application, you may use jQuery event listeners. 1. We avoid adding new jQuery events when they are not required. Instead of adding new jQuery - events take a look at [different methods to do the same task](https://vuejs.org/v2/api/#vm-emit). + events take a look at [different methods to do the same task](https://v2.vuejs.org/v2/api/#vm-emit). 1. You may query the `window` object one time, while bootstrapping your application for application specific data (for example, `scrollTo` is ok to access anytime). Do this access during the bootstrapping of your application. diff --git a/doc/development/fe_guide/tooling.md b/doc/development/fe_guide/tooling.md index 1c32647eefd..2bb6cbfaf7a 100644 --- a/doc/development/fe_guide/tooling.md +++ b/doc/development/fe_guide/tooling.md @@ -175,7 +175,7 @@ preferred editor (all major editors are supported) accordingly. We suggest setting up Prettier to run when each file is saved. For instructions about using Prettier in your preferred editor, see the [Prettier documentation](https://prettier.io/docs/en/editors.html). -Please take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, `.graphql`, and `.scss`). For example, you can exclude file formats in your Visual Studio Code settings file: +Take care that you only let Prettier format the same file types as the global Yarn script does (`.js`, `.vue`, `.graphql`, and `.scss`). For example, you can exclude file formats in your Visual Studio Code settings file: ```json "prettier.disableLanguages": [ diff --git a/doc/development/fe_guide/troubleshooting.md b/doc/development/fe_guide/troubleshooting.md index 14943cca3ac..c0894621ed1 100644 --- a/doc/development/fe_guide/troubleshooting.md +++ b/doc/development/fe_guide/troubleshooting.md @@ -12,7 +12,7 @@ Running into a problem? Maybe this will help ¯\＿(ツ)＿/¯. ### This guide doesn't contain the issue I ran into -If you run into a Frontend development issue that is not in this guide, please consider updating this guide with your issue and possible remedies. This way future adventurers can face these dragons with more success, being armed with your experience and knowledge. +If you run into a Frontend development issue that is not in this guide, consider updating this guide with your issue and possible remedies. This way future adventurers can face these dragons with more success, being armed with your experience and knowledge. ## Testing issues diff --git a/doc/development/fe_guide/view_component.md b/doc/development/fe_guide/view_component.md index f4bb7ac3a2e..2e373e6933b 100644 --- a/doc/development/fe_guide/view_component.md +++ b/doc/development/fe_guide/view_component.md @@ -13,18 +13,24 @@ They are rendered server-side and can be seamlessly used with template languages Refer to the official [documentation](https://viewcomponent.org/) to learn more or watch this [introduction video](https://youtu.be/akRhUbvtnmo). +## Browse components with Lookbook + +We have a [Lookbook](https://github.com/allmarkedup/lookbook) in [http://gdk.test:3000/rails/lookbook](http://gdk.test:3000/rails/lookbook) (only available in development mode) to browse and interact with ViewComponent previews. + ## Pajamas components Some of the components of our [Pajamas](https://design.gitlab.com) design system are available as a ViewComponent in `app/components/pajamas`. NOTE: -We have a small but growing number of Pajamas components. Reach out to the -[Foundations team](https://about.gitlab.com/handbook/engineering/development/dev/ecosystem/foundations/) +We are still in the process of creating these components, so not every Pajamas component is available as ViewComponent. +Reach out to the [Foundations team](https://about.gitlab.com/handbook/engineering/development/dev/ecosystem/foundations/) if the component you are looking for is not yet available. ### Available components +Consider this list a best effort. The full list can be found in [`app/components/pajamas`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/app/components/pajamas). Also see [our Lookbook](http://gdk.test:3000/rails/lookbook) for a more interactive way to browse our components. + #### Alert The `Pajamas::AlertComponent` follows the [Pajamas Alert](https://design.gitlab.com/components/alert) specification. @@ -147,6 +153,39 @@ If you want to add custom attributes to any of these or the card itself, use the For the full list of options, see its [source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/components/pajamas/card_component.rb). +#### Checkbox tag + +The `Pajamas::CheckboxTagComponent` follows the [Pajamas Checkbox](https://design.gitlab.com/components/checkbox) specification. + +The `name` argument and `label` slot are required. + +For example: + +```haml += render Pajamas::CheckboxTagComponent.new(name: 'project[initialize_with_sast]', + checkbox_options: { data: { qa_selector: 'initialize_with_sast_checkbox', track_label: track_label, track_action: 'activate_form_input', track_property: 'init_with_sast' } }) do |c| + = c.label do + = s_('ProjectsNew|Enable Static Application Security Testing (SAST)') + = c.help_text do + = s_('ProjectsNew|Analyze your source code for known security vulnerabilities.') + = link_to _('Learn more.'), help_page_path('user/application_security/sast/index'), target: '_blank', rel: 'noopener noreferrer', data: { track_action: 'followed' } +``` + +For the full list of options, see its +[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/components/pajamas/checkbox_tag_component.rb). + +#### Checkbox + +The `Pajamas::CheckboxComponent` follows the [Pajamas Checkbox](https://design.gitlab.com/components/checkbox) specification. + +NOTE: +`Pajamas::CheckboxComponent` is used internally by the [GitLab UI form builder](haml.md#use-the-gitlab-ui-form-builder) and requires an instance of [ActionView::Helpers::FormBuilder](https://api.rubyonrails.org/v6.1.0/classes/ActionView/Helpers/FormBuilder.html) to be passed as the `form` argument. +It is preferred to use the [gitlab_ui_checkbox_component](haml.md#gitlab_ui_checkbox_component) method to render this ViewComponent. +To use a checkbox without an instance of [ActionView::Helpers::FormBuilder](https://api.rubyonrails.org/v6.1.0/classes/ActionView/Helpers/FormBuilder.html) use [CheckboxTagComponent](#checkbox-tag). + +For the full list of options, see its +[source](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/components/pajamas/checkbox_component.rb). + #### Toggle The `Pajamas::ToggleComponent` follows the [Pajamas Toggle](https://design.gitlab.com/components/toggle) specification. @@ -172,3 +211,5 @@ For the full list of options, see its over creating plain Haml tags with CSS classes. - If you are making changes to an existing Haml view and see, for example, a button that is still implemented with plain Haml, consider migrating it to use a ViewComponent. +- If you decide to create a new component, consider creating [previews](https://viewcomponent.org/guide/previews.html) for it, too. + This will help others to discover your component with Lookbook, also it makes it much easier to test its different states. diff --git a/doc/development/fe_guide/vue.md b/doc/development/fe_guide/vue.md index 7943ae119be..27660c0f5f7 100644 --- a/doc/development/fe_guide/vue.md +++ b/doc/development/fe_guide/vue.md @@ -71,7 +71,7 @@ component, is that you avoid creating a fixture or an HTML element in the unit t ##### `provide` and `inject` -Vue supports dependency injection through [`provide` and `inject`](https://vuejs.org/v2/api/#provide-inject). +Vue supports dependency injection through [`provide` and `inject`](https://v2.vuejs.org/v2/api/#provide-inject). In the component the `inject` configuration accesses the values `provide` passes down. This example of a Vue app initialization shows how the `provide` configuration passes a value from HAML to the component: @@ -266,7 +266,7 @@ return new Vue({ #### Accessing feature flags -Use the [`provide` and `inject`](https://vuejs.org/v2/api/#provide-inject) mechanisms +Use the [`provide` and `inject`](https://v2.vuejs.org/v2/api/#provide-inject) mechanisms in Vue to make feature flags available to any descendant components in a Vue application. The `glFeatures` object is already provided in `commons/vue.js`, so only the mixin is required to use the flags: @@ -339,7 +339,7 @@ Check this [page](vuex.md) for more details. ### Mixing Vue and JavaScript classes (in the data function) -In the [Vue documentation](https://vuejs.org/v2/api/#Options-Data) the Data function/object is defined as follows: +In the [Vue documentation](https://v2.vuejs.org/v2/api/#Options-Data) the Data function/object is defined as follows: > The data object for the Vue instance. Vue recursively converts its properties into getter/setters to make it "reactive". The object must be plain: native objects such as browser API objects and @@ -348,7 +348,7 @@ recommended to observe objects with their own stateful behavior. Based on the Vue guidance: -- **Do not** use or create a JavaScript class in your [data function](https://vuejs.org/v2/api/#data), +- **Do not** use or create a JavaScript class in your [data function](https://v2.vuejs.org/v2/api/#data), such as `user: new User()`. - **Do not** add new JavaScript class implementations. - **Do** use [GraphQL](../api_graphql_styleguide.md), [Vuex](vuex.md) or a set of components if @@ -531,7 +531,7 @@ Each Vue component has a unique output. This output is always present in the ren Although each method of a Vue component can be tested individually, our goal is to test the output of the render function, which represents the state at all times. -Visit the [Vue testing guide](https://vuejs.org/v2/guide/testing.html#Unit-Testing) for help +Visit the [Vue testing guide](https://v2.vuejs.org/v2/guide/testing.html#Unit-Testing) for help testing the rendered output. Here's an example of a well structured unit test for [this Vue component](#appendix---vue-component-subject-under-test): diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index 064f01c8195..8bfb912161a 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -165,7 +165,7 @@ Instead of creating an mutation to toggle the loading state, we should: As a result, we can dispatch the `fetchNamespace` action from the component and it is responsible to commit `REQUEST_NAMESPACE`, `RECEIVE_NAMESPACE_SUCCESS` and `RECEIVE_NAMESPACE_ERROR` mutations. -> Previously, we were dispatching actions from the `fetchNamespace` action instead of committing mutation, so please don't be confused if you find a different pattern in the older parts of the codebase. However, we encourage leveraging a new pattern whenever you write new Vuex stores. +> Previously, we were dispatching actions from the `fetchNamespace` action instead of committing mutation, so don't be confused if you find a different pattern in the older parts of the codebase. However, we encourage leveraging a new pattern whenever you write new Vuex stores. By following this pattern we guarantee: @@ -364,8 +364,8 @@ export default initialState => ({ We made the conscious decision to avoid this pattern to improve the ability to discover and search our frontend codebase. The same applies -when [providing data to a Vue app](vue.md#providing-data-from-haml-to-javascript). The reasoning for this is described in [this -discussion](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/56#note_302514865): +when [providing data to a Vue app](vue.md#providing-data-from-haml-to-javascript). The reasoning for this is described in +[this discussion](https://gitlab.com/gitlab-org/frontend/rfcs/-/issues/56#note_302514865): > Consider a `someStateKey` is being used in the store state. You _may_ not be > able to grep for it directly if it was provided only by `el.dataset`. Instead, diff --git a/doc/development/fe_guide/widgets.md b/doc/development/fe_guide/widgets.md index 02876afe597..b54f9add97d 100644 --- a/doc/development/fe_guide/widgets.md +++ b/doc/development/fe_guide/widgets.md @@ -141,3 +141,7 @@ methods: { ``` [View an example of such a component.](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/notes/components/sidebar_subscription.vue) + +## Merge request widgets + +Refer to the documentation specific to the [merge request widget extension framework](merge_request_widget_extensions.md). diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md index b2d141798fa..a93ed58336d 100644 --- a/doc/development/feature_categorization/index.md +++ b/doc/development/feature_categorization/index.md @@ -10,8 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Each Sidekiq worker, controller action, or API endpoint must declare a `feature_category` attribute. This attribute maps each -of these to a [feature -category](https://about.gitlab.com/handbook/product/categories/). This +of these to a [feature category](https://about.gitlab.com/handbook/product/categories/). This is done for error budgeting, alert routing, and team attribution. The list of feature categories can be found in the file `config/feature_categories.yml`. @@ -29,8 +28,7 @@ product categories. When this occurs, you can automatically update and generate a new version of the file, which needs to be committed to the repository. -The [Scalability -team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/) +The [Scalability team](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/) currently maintains the `feature_categories.yml` file. They will automatically be notified on Slack when the file becomes outdated. diff --git a/doc/development/feature_development.md b/doc/development/feature_development.md index a5d74a0bfd9..e50c1edd282 100644 --- a/doc/development/feature_development.md +++ b/doc/development/feature_development.md @@ -71,9 +71,9 @@ Consult these topics for information on contributing to specific GitLab features - [Developing against interacting components or features](interacting_components.md) - [Manage feature flags](feature_flags/index.md) -- [Licensed feature availability](licensed_feature_availability.md) +- [Implementing Enterprise Edition features](ee_features.md) - [Accessing session data](session.md) -- [How to dump production data to staging](db_dump.md) +- [How to dump production data to staging](database/db_dump.md) - [Geo development](geo.md) - [Redis guidelines](redis.md) - [Adding a new Redis instance](redis/new_redis_instance.md) diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 07c3c83912a..8a862e5f7cd 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -310,7 +310,7 @@ Changes to the issue format can be submitted in the #### Instance level Any feature flag change that affects any GitLab instance is automatically logged in -[features_json.log](../../administration/logs.md#features_jsonlog). +[features_json.log](../../administration/logs/index.md#features_jsonlog). You can search the change history in [Kibana](https://about.gitlab.com/handbook/support/workflows/kibana.html). You can also access the feature flag change history for GitLab.com [in Kibana](https://log.gprd.gitlab.net/goto/d060337c017723084c6d97e09e591fc6). diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 140d5f826cf..502a028f089 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -170,7 +170,7 @@ Each feature flag is defined in a separate YAML file consisting of a number of f | `default_enabled` | yes | The default state of the feature flag. | | `introduced_by_url` | no | The URL to the merge request that introduced the feature flag. | | `rollout_issue_url` | no | The URL to the Issue covering the feature flag rollout. | -| `milestone` | no | Milestone in which the feature was added. | +| `milestone` | no | Milestone in which the feature flag was created. | | `group` | no | The [group](https://about.gitlab.com/handbook/product/categories/#devops-stages) that owns the feature flag. | NOTE: @@ -178,6 +178,9 @@ All validations are skipped when running in `RAILS_ENV=production`. ## Create a new feature flag +NOTE: +GitLab Pages uses [a different process](../pages/index.md#feature-flags) for feature flags. + The GitLab codebase provides [`bin/feature-flag`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/bin/feature-flag), a dedicated tool to create new feature flag definitions. The tool asks various questions about the new feature flag, then creates @@ -423,6 +426,21 @@ Feature.enabled?(:a_feature, project) && Feature.disabled?(:a_feature_override, /chatops run feature set --project=gitlab-org/gitlab a_feature_override true ``` +#### Percentage-based actor selection + +When using the percentage rollout of actors on multiple feature flags, the actors for each feature flag are selected separately. + +For example, the following feature flags are enabled for a certain percentage of actors: + +```plaintext +/chatops run chatops feature set feature-set-1 25 --actors +/chatops run chatops feature set feature-set-2 25 --actors +``` + +If a project A has `:feature-set-1` enabled, there is no guarantee that project A also has `:feature-set-2` enabled. + +For more detail, see [This is how percentages work in Flipper](https://www.hackwithpassion.com/this-is-how-percentages-work-in-flipper). + #### Use actors for verifying in production WARNING: diff --git a/doc/development/features_inside_dot_gitlab.md b/doc/development/features_inside_dot_gitlab.md index ca7dbd6adde..f30a041931e 100644 --- a/doc/development/features_inside_dot_gitlab.md +++ b/doc/development/features_inside_dot_gitlab.md @@ -14,8 +14,8 @@ When implementing new features, please refer to these existing features to avoid - [Merge request Templates](../user/project/description_templates.md#create-a-merge-request-template): `.gitlab/merge_request_templates/`. - [GitLab agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/configuration_repository.md#layout): `.gitlab/agents/`. - [CODEOWNERS](../user/project/code_owners.md#set-up-code-owners): `.gitlab/CODEOWNERS`. -- [Route Maps](../ci/review_apps/#route-maps): `.gitlab/route-map.yml`. +- [Route Maps](../ci/review_apps/index.md#route-maps): `.gitlab/route-map.yml`. - [Customize Auto DevOps Helm Values](../topics/autodevops/customize.md#customize-values-for-helm-chart): `.gitlab/auto-deploy-values.yaml`. - [Insights](../user/project/insights/index.md#configure-your-insights): `.gitlab/insights.yml`. - [Service Desk Templates](../user/project/service_desk.md#using-customized-email-templates): `.gitlab/service_desk_templates/`. -- [Web IDE](../user/project/web_ide/#web-ide-configuration-file): `.gitlab/.gitlab-webide.yml`. +- [Web IDE](../user/project/web_ide/index.md#web-ide-configuration-file): `.gitlab/.gitlab-webide.yml`. diff --git a/doc/development/filtering_by_label.md b/doc/development/filtering_by_label.md index 9e759744a1a..675fe004c22 100644 --- a/doc/development/filtering_by_label.md +++ b/doc/development/filtering_by_label.md @@ -1,179 +1,11 @@ --- -stage: Plan -group: Project Management -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/filtering_by_label.md' +remove_date: '2022-11-05' --- -# Filtering by label -## Introduction +This document was moved to [another location](database/filtering_by_label.md). -GitLab has [labels](../user/project/labels.md) that can be assigned to issues, -merge requests, and epics. Labels on those objects are a many-to-many relation -through the polymorphic `label_links` table. - -To filter these objects by multiple labels - for instance, 'all open -issues with the label ~Plan and the label ~backend' - we generate a -query containing a `GROUP BY` clause. In a simple form, this looks like: - -```sql -SELECT - issues.* -FROM - issues - INNER JOIN label_links ON label_links.target_id = issues.id - AND label_links.target_type = 'Issue' - INNER JOIN labels ON labels.id = label_links.label_id -WHERE - issues.project_id = 13083 - AND (issues.state IN ('opened')) - AND labels.title IN ('Plan', - 'backend') -GROUP BY - issues.id -HAVING (COUNT(DISTINCT labels.title) = 2) -ORDER BY - issues.updated_at DESC, - issues.id DESC -LIMIT 20 OFFSET 0 -``` - -In particular, note that: - -1. We `GROUP BY issues.id` so that we can ... -1. Use the `HAVING (COUNT(DISTINCT labels.title) = 2)` condition to ensure that - all matched issues have both labels. - -This is more complicated than is ideal. It makes the query construction more -prone to errors (such as -[issue #15557](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/15557)). - -## Attempt A: `WHERE EXISTS` - -### Attempt A1: use multiple subqueries with `WHERE EXISTS` - -In [issue #37137](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/37137) -and its associated [merge request](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/14022), -we tried to replace the `GROUP BY` with multiple uses of `WHERE EXISTS`. For the -example above, this would give: - -```sql -WHERE (EXISTS ( - SELECT - TRUE - FROM - label_links - INNER JOIN labels ON labels.id = label_links.label_id - WHERE - labels.title = 'Plan' - AND target_type = 'Issue' - AND target_id = issues.id)) -AND (EXISTS ( - SELECT - TRUE - FROM - label_links - INNER JOIN labels ON labels.id = label_links.label_id - WHERE - labels.title = 'backend' - AND target_type = 'Issue' - AND target_id = issues.id)) -``` - -While this worked without schema changes, and did improve readability somewhat, -it did not improve query performance. - -### Attempt A2: use label IDs in the `WHERE EXISTS` clause - -In [merge request #34503](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34503), we followed a similar approach to A1. But this time, we -did a separate query to fetch the IDs of the labels used in the filter so that we avoid the `JOIN` in the `EXISTS` clause and filter directly by -`label_links.label_id`. We also added a new index on `label_links` for the `target_id`, `label_id`, and `target_type` columns to speed up this query. - -Finding the label IDs wasn't straightforward because there could be multiple labels with the same title within a single root namespace. We solved -this by grouping the label IDs by title and then using the array of IDs in the `EXISTS` clauses. - -This resulted in a significant performance improvement. However, this optimization could not be applied to the dashboard pages -where we do not have a project or group context. We could not easily search for the label IDs here because that would mean searching across all -projects and groups that the user has access to. - -## Attempt B: Denormalize using an array column - -Having [removed MySQL support in GitLab 12.1](https://about.gitlab.com/blog/2019/06/27/removing-mysql-support/), -using [PostgreSQL's arrays](https://www.postgresql.org/docs/11/arrays.html) became more -tractable as we didn't have to support two databases. We discussed denormalizing -the `label_links` table for querying in -[issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651), -with two options: label IDs and titles. - -We can think of both of those as array columns on `issues`, `merge_requests`, -and `epics`: `issues.label_ids` would be an array column of label IDs, and -`issues.label_titles` would be an array of label titles. - -These array columns can be complemented with [GIN -indexes](https://www.postgresql.org/docs/11/gin-intro.html) to improve -matching. - -### Attempt B1: store label IDs for each object - -This has some strong advantages over titles: - -1. Unless a label is deleted, or a project is moved, we never need to - bulk-update the denormalized column. -1. It uses less storage than the titles. - -Unfortunately, our application design makes this hard. If we were able to query -just by label ID easily, we wouldn't need the `INNER JOIN labels` in the initial -query at the start of this document. GitLab allows users to filter by label -title across projects and even across groups, so a filter by the label ~Plan may -include labels with multiple distinct IDs. - -We do not want users to have to know about the different IDs, which means that -given this data set: - -| Project | ~Plan label ID | ~backend label ID | -| ------- | -------------- | ----------------- | -| A | 11 | 12 | -| B | 21 | 22 | -| C | 31 | 32 | - -We would need something like: - -```sql -WHERE - label_ids @> ARRAY[11, 12] - OR label_ids @> ARRAY[21, 22] - OR label_ids @> ARRAY[31, 32] -``` - -This can get even more complicated when we consider that in some cases, there -might be two ~backend labels - with different IDs - that could apply to the same -object, so the number of combinations would balloon further. - -### Attempt B2: store label titles for each object - -From the perspective of updating the object, this is the worst -option. We have to bulk update the objects when: - -1. The objects are moved from one project to another. -1. The project is moved from one group to another. -1. The label is renamed. -1. The label is deleted. - -It also uses much more storage. Querying is simple, though: - -```sql -WHERE - label_titles @> ARRAY['Plan', 'backend'] -``` - -And our [tests in issue #49651](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/49651#note_188777346) -showed that this could be fast. - -However, at present, the disadvantages outweigh the advantages. - -## Conclusion - -We found a method A2 that does not need denormalization and improves the query performance significantly. This -did not apply to all cases, but we were able to apply method A1 to the rest of the cases so that we remove the -`GROUP BY` and `HAVING` clauses in all scenarios. - -This simplified the query and improved the performance in the most common cases. +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index 6261b2fda6f..c690408ee60 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -12,12 +12,9 @@ to ensure a certain security floor is met by vendors selling products to U.S. Federal institutions. WARNING: -GitLab is not FIPS compliant, even when built and run on a FIPS-enforcing -system. Large parts of the build are broken, and many features use forbidden -cryptographic primitives. Running GitLab on a FIPS-enforcing system is not -supported and may result in data loss. This document is intended to help -engineers looking to develop FIPS-related fixes. It is not intended to be used -to run a production GitLab instance. +You can build a FIPS-compliant instance of GitLab, but [not all features are included](#unsupported-features-in-fips-mode). +A FIPS-compliant instance must be configured following the [FIPS install instructions](#install-gitlab-with-fips-compliance) +exactly. There are two current FIPS standards: [140-2](https://en.wikipedia.org/wiki/FIPS_140-2) and [140-3](https://en.wikipedia.org/wiki/FIPS_140-3). At GitLab we usually @@ -25,10 +22,7 @@ mean FIPS 140-2. ## Current status -Read [Epic &5104](https://gitlab.com/groups/gitlab-org/-/epics/5104) for more -information on the status of the investigation. - -GitLab is actively working towards FIPS compliance. +GitLab is actively working towards FIPS compliance. Progress on this initiative can be tracked with this [FIPS compliance Epic](https://gitlab.com/groups/gitlab-org/-/epics/6452). ## FIPS compliance at GitLab @@ -46,15 +40,40 @@ when FIPS mode is enabled. | Ubuntu 20.04 Libgcrypt Cryptographic Module | [3902](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3902) | EC2 instances | `gpg`, `sshd` | | Amazon Linux 2 Kernel Crypto API Cryptographic Module | [3709](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3709) | EKS nodes | Linux kernel | | Amazon Linux 2 OpenSSL Cryptographic Module | [3553](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3553) | EKS nodes | NGINX | -| RedHat Enterprise Linux 8 OpenSSL Cryptographic Module | [3852](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3852) | EKS nodes | UBI containers: Workhorse, Pages, Container Registry, Rails (Puma/Sidekiq), Security Analyzers | +| RedHat Enterprise Linux 8 OpenSSL Cryptographic Module | [4271](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/4271) | EKS nodes | UBI containers: Workhorse, Pages, Container Registry, Rails (Puma/Sidekiq), Security Analyzers | | RedHat Enterprise Linux 8 Libgcrypt Cryptographic Module | [3784](https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/3784) | EKS nodes | UBI containers: GitLab Shell, `gpg` | ### Supported Operating Systems -The supported hybrid environments are: +The supported hybrid platforms are: + +- Omnibus GitLab: Ubuntu 20.04 LTS +- Cloud Native GitLab: Amazon Linux 2 (EKS) + +### Unsupported features in FIPS mode + +Some GitLab features may not work when FIPS mode is enabled. The following features +are known to not work in FIPS mode. However, there may be additional features not +listed here that also do not work properly in FIPS mode: + +- [Container Scanning](../user/application_security/container_scanning/index.md) support for scanning images in repositories that require authentication. +- [Code Quality](../ci/testing/code_quality.md) does not support operating in FIPS-compliant mode. +- [Dependency scanning](../user/application_security/dependency_scanning/index.md) support for Gradle. +- [Dynamic Application Security Testing (DAST)](../user/application_security/dast/index.md) + does not support operating in FIPS-compliant mode. +- [License compliance](../user/compliance/license_compliance/index.md). +- [Solutions for vulnerabilities](../user/application_security/vulnerabilities/index.md#resolve-a-vulnerability) + for yarn projects. +- [Static Application Security Testing (SAST)](../user/application_security/sast/index.md) + supports a reduced set of [analyzers](../user/application_security/sast/#fips-enabled-images) + when operating in FIPS-compliant mode. +- Advanced Search is currently not included in FIPS mode. It must not be enabled in order to be FIPS-compliant. +- [Gravatar or Libravatar-based profile images](../administration/libravatar.md) are not FIPS-compliant. + +Additionally, these package repositories are disabled in FIPS mode: -- Omnibus: Ubuntu 20.04 FIPS -- EKS: Amazon Linux 2 +- [Conan package repository](../user/packages/conan_repository/index.md). +- [Debian package repository](../user/packages/debian_repository/index.md). ## FIPS validation at GitLab @@ -281,6 +300,9 @@ gitlab: gitlab-mailroom: image: tag: master-fips + gitlab-pages: + image: + tag: master-fips migrations: image: tag: master-fips @@ -299,18 +321,17 @@ gitlab: nginx-ingress: controller: image: - repository: registry.gitlab.com/stanhu/gitlab-test-images/k8s-staging-ingress-nginx/controller - tag: v1.2.0-beta.1 + repository: registry.gitlab.com/gitlab-org/cloud-native/charts/gitlab-ingress-nginx/controller + tag: v1.2.1-fips pullPolicy: Always - digest: sha256:ace38833689ad34db4a46bc1e099242696eb800def88f02200a8615530734116 + digest: sha256:c4222b7ab3836b9be2a7649cff4b2e6ead34286dfdf3a7b04eb34fdd3abb0334 ``` The above example shows a FIPS-enabled [`nginx-ingress`](https://github.com/kubernetes/ingress-nginx) image. -See [this issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3153#note_917782207) for more details on -how to build NGINX and the Ingress Controller. +See our [Charts documentation on FIPS](https://docs.gitlab.com/charts/advanced/fips/index.html) for more details. You can also use release tags, but the versioning is tricky because each -component may use its own versioning scheme. For example, for GitLab v15.1: +component may use its own versioning scheme. For example, for GitLab v15.2: ```yaml global: @@ -324,30 +345,33 @@ global: gitlab: gitaly: image: - tag: v15.1.0-fips + tag: v15.2.0-fips gitlab-exporter: image: - tag: 11.15.2-fips + tag: 11.17.1-fips gitlab-shell: image: - tag: v15.1.0-fips + tag: v14.9.0-fips gitlab-mailroom: image: - tag: v15.1.0-fips + tag: v15.2.0-fips + gitlab-pages: + image: + tag: v1.61.0-fips migrations: image: - tag: v15.1.0-fips + tag: v15.2.0-fips sidekiq: image: - tag: v15.1.0-fips + tag: v15.2.0-fips toolbox: image: - tag: v15.1.0-fips + tag: v15.2.0-fips webservice: image: - tag: v15.1.0-fips + tag: v15.2.0-fips workhorse: - tag: v15.1.0-fips + tag: v15.2.0-fips ``` ## FIPS Performance Benchmarking @@ -417,29 +441,6 @@ def default_min_key_size(name) end ``` -#### Unsupported features in FIPS mode - -Some GitLab features may not work when FIPS mode is enabled. The following features -are known to not work in FIPS mode. However, there may be additional features not -listed here that also do not work properly in FIPS mode: - -- [Container Scanning](../user/application_security/container_scanning/index.md) support for scanning images in repositories that require authentication. -- [Code Quality](../ci/testing/code_quality.md) does not support operating in FIPS-compliant mode. -- [Dependency scanning](../user/application_security/dependency_scanning/index.md) support for Gradle. -- [Dynamic Application Security Testing (DAST)](../user/application_security/dast/index.md) - does not support operating in FIPS-compliant mode. -- [License compliance](../user/compliance/license_compliance/index.md). -- [Solutions for vulnerabilities](../user/application_security/vulnerabilities/index.md#resolve-a-vulnerability) - for yarn projects. -- [Static Application Security Testing (SAST)](../user/application_security/sast/index.md) - supports a reduced set of [analyzers](../user/application_security/sast/#fips-enabled-images) - when operating in FIPS-compliant mode. - -Additionally, these package repositories are disabled in FIPS mode: - -- [Conan package repository](../user/packages/conan_repository/index.md). -- [Debian package repository](../user/packages/debian_repository/index.md). - ## Nightly Omnibus FIPS builds The Distribution team has created [nightly FIPS Omnibus builds](https://packages.gitlab.com/gitlab/nightly-fips-builds). These diff --git a/doc/development/foreign_keys.md b/doc/development/foreign_keys.md index e0dd0fe8e7c..cdf655bf0bf 100644 --- a/doc/development/foreign_keys.md +++ b/doc/development/foreign_keys.md @@ -1,200 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/foreign_keys.md' +remove_date: '2022-11-05' --- -# Foreign Keys & Associations +This document was moved to [another location](database/foreign_keys.md). -When adding an association to a model you must also add a foreign key. For -example, say you have the following model: - -```ruby -class User < ActiveRecord::Base - has_many :posts -end -``` - -Add a foreign key here on column `posts.user_id`. This ensures -that data consistency is enforced on database level. Foreign keys also mean that -the database can very quickly remove associated data (for example, when removing a -user), instead of Rails having to do this. - -## Adding Foreign Keys In Migrations - -Foreign keys can be added concurrently using `add_concurrent_foreign_key` as -defined in `Gitlab::Database::MigrationHelpers`. See the [Migration Style -Guide](migration_style_guide.md) for more information. - -Keep in mind that you can only safely add foreign keys to existing tables after -you have removed any orphaned rows. The method `add_concurrent_foreign_key` -does not take care of this so you must do so manually. See -[adding foreign key constraint to an existing column](database/add_foreign_key_to_existing_column.md). - -## Updating Foreign Keys In Migrations - -Sometimes a foreign key constraint must be changed, preserving the column -but updating the constraint condition. For example, moving from -`ON DELETE CASCADE` to `ON DELETE SET NULL` or vice-versa. - -PostgreSQL does not prevent you from adding overlapping foreign keys. It -honors the most recently added constraint. This allows us to replace foreign keys without -ever losing foreign key protection on a column. - -To replace a foreign key: - -1. [Add the new foreign key without validation](database/add_foreign_key_to_existing_column.md#prevent-invalid-records) - - The name of the foreign key constraint must be changed to add a new - foreign key before removing the old one. - - ```ruby - class ReplaceFkOnPackagesPackagesProjectId < Gitlab::Database::Migration[2.0] - disable_ddl_transaction! - - NEW_CONSTRAINT_NAME = 'fk_new' - - def up - add_concurrent_foreign_key(:packages_packages, :projects, column: :project_id, on_delete: :nullify, validate: false, name: NEW_CONSTRAINT_NAME) - end - - def down - with_lock_retries do - remove_foreign_key_if_exists(:packages_packages, column: :project_id, on_delete: :nullify, name: NEW_CONSTRAINT_NAME) - end - end - end - ``` - -1. [Validate the new foreign key](database/add_foreign_key_to_existing_column.md#validate-the-foreign-key) - - ```ruby - class ValidateFkNew < Gitlab::Database::Migration[2.0] - NEW_CONSTRAINT_NAME = 'fk_new' - - # foreign key added in <link to MR or path to migration adding new FK> - def up - validate_foreign_key(:packages_packages, name: NEW_CONSTRAINT_NAME) - end - - def down - # no-op - end - end - ``` - -1. Remove the old foreign key: - - ```ruby - class RemoveFkOld < Gitlab::Database::Migration[2.0] - OLD_CONSTRAINT_NAME = 'fk_old' - - # new foreign key added in <link to MR or path to migration adding new FK> - # and validated in <link to MR or path to migration validating new FK> - def up - remove_foreign_key_if_exists(:packages_packages, column: :project_id, on_delete: :cascade, name: OLD_CONSTRAINT_NAME) - end - - def down - # Validation is skipped here, so if rolled back, this will need to be revalidated in a separate migration - add_concurrent_foreign_key(:packages_packages, :projects, column: :project_id, on_delete: :cascade, validate: false, name: OLD_CONSTRAINT_NAME) - end - end - ``` - -## Cascading Deletes - -Every foreign key must define an `ON DELETE` clause, and in 99% of the cases -this should be set to `CASCADE`. - -## Indexes - -When adding a foreign key in PostgreSQL the column is not indexed automatically, -thus you must also add a concurrent index. Not doing so results in cascading -deletes being very slow. - -## Naming foreign keys - -By default Ruby on Rails uses the `_id` suffix for foreign keys. So we should -only use this suffix for associations between two tables. If you want to -reference an ID on a third party platform the `_xid` suffix is recommended. - -The spec `spec/db/schema_spec.rb` tests if all columns with the `_id` suffix -have a foreign key constraint. So if that spec fails, don't add the column to -`IGNORED_FK_COLUMNS`, but instead add the FK constraint, or consider naming it -differently. - -## Dependent Removals - -Don't define options such as `dependent: :destroy` or `dependent: :delete` when -defining an association. Defining these options means Rails handles the -removal of data, instead of letting the database handle this in the most -efficient way possible. - -In other words, this is bad and should be avoided at all costs: - -```ruby -class User < ActiveRecord::Base - has_many :posts, dependent: :destroy -end -``` - -Should you truly have a need for this it should be approved by a database -specialist first. - -You should also not define any `before_destroy` or `after_destroy` callbacks on -your models _unless_ absolutely required and only when approved by database -specialists. For example, if each row in a table has a corresponding file on a -file system it may be tempting to add a `after_destroy` hook. This however -introduces non database logic to a model, and means we can no longer rely on -foreign keys to remove the data as this would result in the file system data -being left behind. In such a case you should use a service class instead that -takes care of removing non database data. - -In cases where the relation spans multiple databases you have even -further problems using `dependent: :destroy` or the above hooks. You can -read more about alternatives at [Avoid `dependent: :nullify` and -`dependent: :destroy` across -databases](database/multiple_databases.md#avoid-dependent-nullify-and-dependent-destroy-across-databases). - -## Alternative primary keys with `has_one` associations - -Sometimes a `has_one` association is used to create a one-to-one relationship: - -```ruby -class User < ActiveRecord::Base - has_one :user_config -end - -class UserConfig < ActiveRecord::Base - belongs_to :user -end -``` - -In these cases, there may be an opportunity to remove the unnecessary `id` -column on the associated table, `user_config.id` in this example. Instead, -the originating table ID can be used as the primary key for the associated -table: - -```ruby -create_table :user_configs, id: false do |t| - t.references :users, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade } - ... -end -``` - -Setting `default: nil` ensures a primary key sequence is not created, and since the primary key -automatically gets an index, we set `index: false` to avoid creating a duplicate. -You also need to add the new primary key to the model: - -```ruby -class UserConfig < ActiveRecord::Base - self.primary_key = :user_id - - belongs_to :user -end -``` - -Using a foreign key as primary key saves space but can make -[batch counting](service_ping/implement.md#batch-counters) in [Service Ping](service_ping/index.md) less efficient. -Consider using a regular `id` column if the table is relevant for Service Ping. +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/gemfile.md b/doc/development/gemfile.md index 0fcfb88c9cd..f9cf69020bb 100644 --- a/doc/development/gemfile.md +++ b/doc/development/gemfile.md @@ -61,8 +61,7 @@ to a gem, go through these steps: 1. Follow the [instructions for new projects](https://about.gitlab.com/handbook/engineering/gitlab-repositories/#creating-a-new-project). 1. Follow the instructions for setting up a [CI/CD configuration](https://about.gitlab.com/handbook/engineering/gitlab-repositories/#cicd-configuration). 1. Follow the instructions for [publishing a project](https://about.gitlab.com/handbook/engineering/gitlab-repositories/#publishing-a-project). - - See [issue - #325463](https://gitlab.com/gitlab-org/gitlab/-/issues/325463) + - See [issue #325463](https://gitlab.com/gitlab-org/gitlab/-/issues/325463) for an example. - In some cases we may want to move a gem to its own namespace. Some examples might be that it will naturally have more than one project @@ -74,8 +73,8 @@ to a gem, go through these steps: apply if someone who currently works at GitLab wants to maintain the gem beyond their time working at GitLab. -When publishing a gem to RubyGems.org, also note the section on [gem -owners](https://about.gitlab.com/handbook/developer-onboarding/#ruby-gems) +When publishing a gem to RubyGems.org, also note the section on +[gem owners](https://about.gitlab.com/handbook/developer-onboarding/#ruby-gems) in the handbook. ## Upgrade Rails @@ -113,8 +112,7 @@ gem 'thor', '>= 1.1.1' ``` Here we're using the operator `>=` (greater than or equal to) rather -than `~>` ([pessimistic -operator](https://thoughtbot.com/blog/rubys-pessimistic-operator)) +than `~>` ([pessimistic operator](https://thoughtbot.com/blog/rubys-pessimistic-operator)) making it possible to upgrade `license_finder` or any other gem to a version that depends on `thor 1.2`. @@ -134,15 +132,14 @@ that also relied on `thor` but had its version pinned to a vulnerable one. These changes are easy to miss in the `Gemfile.lock`. Pinning the version would result in a conflict that would need to be solved. -To avoid upgrading indirect dependencies, we can use [`bundle update ---conservative`](https://bundler.io/man/bundle-update.1.html#OPTIONS). +To avoid upgrading indirect dependencies, we can use +[`bundle update --conservative`](https://bundler.io/man/bundle-update.1.html#OPTIONS). When submitting a merge request including a dependency update, include a link to the Gem diff between the 2 versions in the merge request description. You can find this link on `rubygems.org`, select **Review Changes** to generate a comparison between the versions on `diffend.io`. For example, this is the gem -diff for [`thor` 1.0.0 vs -1.0.1](https://my.diffend.io/gems/thor/1.0.0/1.0.1). Use the +diff for [`thor` 1.0.0 vs 1.0.1](https://my.diffend.io/gems/thor/1.0.0/1.0.1). Use the links directly generated from RubyGems, since the links from GitLab or other code-hosting platforms might not reflect the code that's actually published. diff --git a/doc/development/geo.md b/doc/development/geo.md index 9e9bd85ecd8..f042af42de5 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -576,7 +576,7 @@ See `Gitlab::Geo.enabled?` and `Gitlab::Geo.license_allows?` methods. All Geo **secondary** sites are read-only. -The general principle of a [read-only database](verifying_database_capabilities.md#read-only-database) +The general principle of a [read-only database](database/verifying_database_capabilities.md#read-only-database) applies to all Geo **secondary** sites. So the `Gitlab::Database.read_only?` method will always return `true` on a **secondary** site. diff --git a/doc/development/geo/proxying.md b/doc/development/geo/proxying.md index 41c7f426c6f..2f0226c489c 100644 --- a/doc/development/geo/proxying.md +++ b/doc/development/geo/proxying.md @@ -128,8 +128,8 @@ Secondary-->>Client: admin/geo/replication/projects logged in response (session ## Git pull -For historical reasons, the `push_from_secondary` path is used to forward a Git pull. There is [an issue proposing to -rename this route](https://gitlab.com/gitlab-org/gitlab/-/issues/292690) to avoid confusion. +For historical reasons, the `push_from_secondary` path is used to forward a Git pull. There is +[an issue proposing to rename this route](https://gitlab.com/gitlab-org/gitlab/-/issues/292690) to avoid confusion. ### Git pull over HTTP(s) diff --git a/doc/development/git_object_deduplication.md b/doc/development/git_object_deduplication.md index 1a864ef81f0..a6b359769f8 100644 --- a/doc/development/git_object_deduplication.md +++ b/doc/development/git_object_deduplication.md @@ -18,8 +18,8 @@ GitLab implements Git object deduplication. ### Understanding Git alternates -At the Git level, we achieve deduplication by using [Git -alternates](https://git-scm.com/docs/gitrepository-layout#gitrepository-layout-objects). +At the Git level, we achieve deduplication by using +[Git alternates](https://git-scm.com/docs/gitrepository-layout#gitrepository-layout-objects). Git alternates is a mechanism that lets a repository borrow objects from another repository on the same machine. @@ -44,8 +44,8 @@ reliable decide if an object is no longer needed. ### Git alternates in GitLab: pool repositories -GitLab organizes this object borrowing by [creating special **pool -repositories**](../administration/repository_storage_types.md) which are hidden from the user. We then use Git +GitLab organizes this object borrowing by [creating special **pool repositories**](../administration/repository_storage_types.md) +which are hidden from the user. We then use Git alternates to let a collection of project repositories borrow from a single pool repository. We call such a collection of project repositories a pool. Pools form star-shaped networks of repositories @@ -99,9 +99,8 @@ are as follows: ### Assumptions -- All repositories in a pool must use [hashed - storage](../administration/repository_storage_types.md). This is so - that we don't have to ever worry about updating paths in +- All repositories in a pool must use [hashed storage](../administration/repository_storage_types.md). + This is so that we don't have to ever worry about updating paths in `object/info/alternates` files. - All repositories in a pool must be on the same Gitaly storage shard. The Git alternates mechanism relies on direct disk access across diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 8a0cf8e7717..66b535682f5 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -79,8 +79,7 @@ During RSpec tests, the Gitaly instance writes logs to `gitlab/log/gitaly-test.l While Gitaly can handle all Git access, many of GitLab customers still run Gitaly atop NFS. The legacy Rugged implementation for Git calls may be faster than the Gitaly RPC due to N+1 Gitaly calls and other -reasons. See [the -issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57317) for more +reasons. See [the issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57317) for more details. Until GitLab has eliminated most of these inefficiencies or the use of diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index 57cb74a6159..0aa1bad711d 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -71,8 +71,8 @@ This worker imports all pull requests. For every pull request a job for the ### 5. Stage::ImportPullRequestsMergedByWorker -This worker imports the pull requests' _merged-by_ user information. The [_List pull -requests_](https://docs.github.com/en/rest/reference/pulls#list-pull-requests) +This worker imports the pull requests' _merged-by_ user information. The +[_List pull requests_](https://docs.github.com/en/rest/pulls#list-pull-requests) API doesn't provide this information. Therefore, this stage must fetch each merged pull request individually to import this information. A `Gitlab::GithubImport::ImportPullRequestMergedByWorker` job is scheduled for each fetched pull diff --git a/doc/development/gitlab_flavored_markdown/specification_guide/index.md b/doc/development/gitlab_flavored_markdown/specification_guide/index.md index 80837506037..756b87cd407 100644 --- a/doc/development/gitlab_flavored_markdown/specification_guide/index.md +++ b/doc/development/gitlab_flavored_markdown/specification_guide/index.md @@ -54,7 +54,7 @@ simultaneous in the same areas of logic. In these situations, _GitHub_ Flavored Markdown may be referred to with variable or constant names like `ghfm_` to avoid confusion. For example, we use the `ghfm` acronym for the [`ghfm_spec_v_0.29.txt` GitHub Flavored Markdown specification file](#github-flavored-markdown-specification) -which is committed to the `gitlab` repo and used as input to the +which is committed to the `gitlab` repository and used as input to the [`update_specification.rb` script](#update-specificationrb-script). The original CommonMark specification is referred to as _CommonMark_ (no acronym). @@ -86,6 +86,51 @@ it does not have a static, hardcoded, manually updated `spec.txt`. Instead, the GLFM `spec.txt` is automatically generated based on other input files. This process is explained in detail in the [Implementation](#implementation) sections below. +#### Official specifications vs internal extensions + +Within GFM and GLFM respectively, both GitHub and GitLab have two "sets" of Markdown they support: + +- Official specification +- Internal extensions + +The following taxonomy chart shows the taxonomy and terminology of the various specifications: + +```mermaid +graph TD +CM[CommonMark - spec.txt - e.g. headings] --- GFMS[GFM Specification - spec.txt - e.g. strikethrough extension] +GFMS --- GLFM[GLFM Specification - e.g. color chips] +GFMS --- GFMI[GFM internal extensions - e.g. GitHub-specific references] +GLFM --- GLFS[GLFM internal extensions - e.g. GitLab-specific references] +``` + +##### Official specifications + +GFM and GLFM each have an official specification, which includes both: + +1. The CommonMark standard. +1. Generic extensions to the CommonMark standard. + +For example, GFM adds the +[`strikethrough` extension](https://github.github.com/gfm/#strikethrough-extension-), +and GLFM adds the +[`color chips` extension](../../../user/markdown.md#colors). +These extensions in the official specifications are not dependent upon any specific +implementation or environment. They can be implemented in any third-party Markdown rendering engine. + +##### Internal extensions + +GFM and GLFM each also have a set of internal extensions. These extensions are not part of the GFM or GLFM +official specifications, but are part of the GitHub and GitLab internal Markdown renderer and parser +implementations. These internal extensions are often dependent upon the GitHub or GitLab +implementations or environments, and may depend upon metadata which is only available via +interacting with those environments. For example, +[GitHub supports GitHub-specific autolinked references](https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/autolinked-references-and-urls), +and +[GitLab also supports GitLab-specific references](../../../user/markdown.md#gitlab-specific-references). +These may also be implemented by third-party Markdown rendering engines which integrate with +GitHub or GitLab. For example, editor or IDE plugins which enable the user to directly edit +Markdown for issues, pull requests, or merge requests within the editor or IDE. + ### Markdown examples Everywhere in the context of the specification and this guide, the term @@ -136,7 +181,7 @@ NOTE: #### Markdown snapshot testing _Markdown snapshot testing_ refers to the automated testing performed in -the GitLab codebase, which is driven by "example_snapshots" fixture data derived from all of +the GitLab codebase, which is driven by `example_snapshots` fixture data derived from all of the examples in the GLFM specification. It consists of both backend RSpec tests and frontend Jest tests which use the fixture data. This fixture data is contained in YAML files. These files are generated and updated based on the Markdown examples in the specification, diff --git a/doc/development/go_guide/dependencies.md b/doc/development/go_guide/dependencies.md index 0c2ce4f2b48..2a53fa590e3 100644 --- a/doc/development/go_guide/dependencies.md +++ b/doc/development/go_guide/dependencies.md @@ -44,9 +44,9 @@ end with a timestamp and the first 12 characters of the commit identifier: If a VCS tag matches one of these patterns, it is ignored. -For a complete understanding of Go modules and versioning, see [this series of -blog posts](https://go.dev/blog/using-go-modules) on the official Go -website. +For a complete understanding of Go modules and versioning, see +[this series of blog posts](https://go.dev/blog/using-go-modules) +on the official Go website. ## 'Module' vs 'Package' diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 1a11321b70f..711b0662a8c 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -145,18 +145,16 @@ Go GitLab linter plugins are maintained in the [`gitlab-org/language-tools/go/li ## Dependencies Dependencies should be kept to the minimum. The introduction of a new -dependency should be argued in the merge request, as per our [Approval -Guidelines](../code_review.md#approval-guidelines). Both [License -Scanning](../../user/compliance/license_compliance/index.md) -**(ULTIMATE)** and [Dependency -Scanning](../../user/application_security/dependency_scanning/index.md) -**(ULTIMATE)** should be activated on all projects to ensure new dependencies +dependency should be argued in the merge request, as per our [Approval Guidelines](../code_review.md#approval-guidelines). +Both [License Scanning](../../user/compliance/license_compliance/index.md) +and [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) +should be activated on all projects to ensure new dependencies security status and license compatibility. ### Modules -In Go 1.11 and later, a standard dependency system is available behind the name [Go -Modules](https://github.com/golang/go/wiki/Modules). It provides a way to +In Go 1.11 and later, a standard dependency system is available behind the name +[Go Modules](https://github.com/golang/go/wiki/Modules). It provides a way to define and lock dependencies for reproducible builds. It should be used whenever possible. @@ -168,8 +166,8 @@ projects, and makes merge requests easier to review. In some cases, such as building a Go project for it to act as a dependency of a CI run for another project, removing the `vendor/` directory means the code must be downloaded repeatedly, which can lead to intermittent problems due to rate -limiting or network failures. In these circumstances, you should [cache the -downloaded code between](../../ci/caching/index.md#cache-go-dependencies). +limiting or network failures. In these circumstances, you should +[cache the downloaded code between](../../ci/caching/index.md#cache-go-dependencies). There was a [bug on modules checksums](https://github.com/golang/go/issues/29278) in Go versions earlier than v1.11.4, so make @@ -330,18 +328,15 @@ A few things to keep in mind when adding context: ### References for working with errors - [Go 1.13 errors](https://go.dev/blog/go1.13-errors). -- [Programing with - errors](https://peter.bourgon.org/blog/2019/09/11/programming-with-errors.html). -- [Don't just check errors, handle them - gracefully](https://dave.cheney.net/2016/04/27/dont-just-check-errors-handle-them-gracefully). +- [Programing with errors](https://peter.bourgon.org/blog/2019/09/11/programming-with-errors.html). +- [Don't just check errors, handle them gracefully](https://dave.cheney.net/2016/04/27/dont-just-check-errors-handle-them-gracefully). ## CLIs Every Go program is launched from the command line. [`cli`](https://github.com/urfave/cli) is a convenient package to create command line apps. It should be used whether the project is a daemon or a simple CLI -tool. Flags can be mapped to [environment -variables](https://github.com/urfave/cli#values-from-the-environment) directly, +tool. Flags can be mapped to [environment variables](https://github.com/urfave/cli#values-from-the-environment) directly, which documents and centralizes at the same time all the possible command line interactions with the program. Don't use `os.GetEnv`, it hides variables deep in the code. @@ -362,8 +357,7 @@ Every binary ideally must have structured (JSON) logging in place as it helps with searching and filtering the logs. At GitLab we use structured logging in JSON format, as all our infrastructure assumes that. When using [Logrus](https://github.com/sirupsen/logrus) you can turn on structured -logging simply by using the build in [JSON -formatter](https://github.com/sirupsen/logrus#formatters). This follows the +logging simply by using the build in [JSON formatter](https://github.com/sirupsen/logrus#formatters). This follows the same logging type we use in our [Ruby applications](../logging.md#use-structured-json-logging). #### How to use Logrus @@ -414,8 +408,7 @@ should be used in functions that can block and passed as the first parameter. Every project should have a `Dockerfile` at the root of their repository, to build and run the project. Since Go program are static binaries, they should not require any external dependency, and shells in the final image are useless. -We encourage [Multistage -builds](https://docs.docker.com/develop/develop-images/multistage-build/): +We encourage [Multistage builds](https://docs.docker.com/develop/develop-images/multistage-build/): - They let the user build the project with the right Go version and dependencies. @@ -448,36 +441,28 @@ up to run `goimports -local gitlab.com/gitlab-org` so that it's applied to every If initializing a slice, provide a capacity where possible to avoid extra allocations. -<table> -<tr><th>:white_check_mark: Do</th><th>:x: Don't</th></tr> -<tr> - <td> +**Don't:** - ```golang - s2 := make([]string, 0, size) - for _, val := range s1 { - s2 = append(s2, val) - } - ``` +```golang +var s2 []string +for _, val := range s1 { + s2 = append(s2, val) +} +``` - </td> - <td> +**Do:** - ```golang - var s2 []string - for _, val := range s1 { - s2 = append(s2, val) - } - ``` - - </td> -</tr> -</table> +```golang +s2 := make([]string, 0, size) +for _, val := range s1 { + s2 = append(s2, val) +} +``` If no capacity is passed to `make` when creating a new slice, `append` will continuously resize the slice's backing array if it cannot hold the values. Providing the capacity ensures that allocations are kept -to a minimum. It is recommended that the [`prealloc`](https://github.com/alexkohler/prealloc) +to a minimum. It's recommended that the [`prealloc`](https://github.com/alexkohler/prealloc) golanci-lint rule automatically check for this. ### Analyzer Tests diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index d89dbbcf904..af11340737f 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -200,8 +200,7 @@ refresh_service.execute(oldrev, newrev, ref) See ["Why is it bad style to `rescue Exception => e` in Ruby?"](https://stackoverflow.com/questions/10048173/why-is-it-bad-style-to-rescue-exception-e-in-ruby). -This rule is [enforced automatically by -RuboCop](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-4-stable/.rubocop.yml#L911-914)._ +This rule is [enforced automatically by RuboCop](https://gitlab.com/gitlab-org/gitlab-foss/blob/8-4-stable/.rubocop.yml#L911-914)._ ## Do not use inline JavaScript in views diff --git a/doc/development/hash_indexes.md b/doc/development/hash_indexes.md index 731639b6f06..2a9f7e5a25d 100644 --- a/doc/development/hash_indexes.md +++ b/doc/development/hash_indexes.md @@ -1,26 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/hash_indexes.md' +remove_date: '2022-11-06' --- -# Hash Indexes +This document was moved to [another location](database/hash_indexes.md). -PostgreSQL supports hash indexes besides the regular B-tree -indexes. Hash indexes however are to be avoided at all costs. While they may -_sometimes_ provide better performance the cost of rehashing can be very high. -More importantly: at least until PostgreSQL 10.0 hash indexes are not -WAL-logged, meaning they are not replicated to any replicas. From the PostgreSQL -documentation: - -> Hash index operations are not presently WAL-logged, so hash indexes might need -> to be rebuilt with REINDEX after a database crash if there were unwritten -> changes. Also, changes to hash indexes are not replicated over streaming or -> file-based replication after the initial base backup, so they give wrong -> answers to queries that subsequently use them. For these reasons, hash index -> use is presently discouraged. - -RuboCop is configured to register an offense when it detects the use of a hash -index. - -Instead of using hash indexes you should use regular B-tree indexes. +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 18704fc2b60..ea22b33f1bc 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -85,7 +85,7 @@ Or: hello = _("Hello world!") ``` -Be careful when translating strings at the class or module level since these are only evaluated once +Be careful when translating strings at the class or module level because these are only evaluated once at class load time. For example: ```ruby @@ -299,16 +299,16 @@ use `%{created_at}` in Ruby but `%{createdAt}` in JavaScript. Make sure to - In Ruby/HAML: ```ruby - _("Hello %{name}") % { name: 'Joe' } => 'Hello Joe' + format(_("Hello %{name}"), name: 'Joe') => 'Hello Joe' ``` - In Vue: Use the [`GlSprintf`](https://gitlab-org.gitlab.io/gitlab-ui/?path=/docs/utilities-sprintf--sentence-with-link) component if: - - You need to include child components in the translation string. - - You need to include HTML in your translation string. - - You're using `sprintf` and need to pass `false` as the third argument to + - You are including child components in the translation string. + - You are including HTML in your translation string. + - You are using `sprintf` and are passing `false` as the third argument to prevent it from escaping placeholder values. For example: @@ -482,7 +482,7 @@ Instead of this: ```ruby # incorrect usage example -n_("%{project_name}", "%d projects selected", count) % { project_name: 'GitLab' } +format(n_("%{project_name}", "%d projects selected", count), project_name: 'GitLab') ``` ### Namespaces diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index cee078ca891..f986a852567 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -34,6 +34,7 @@ are very appreciative of the work done by translators and proofreaders! - Weizhe Ding - [GitLab](https://gitlab.com/d.weizhe), [Crowdin](https://crowdin.com/profile/d.weizhe) - Yi-Jyun Pan - [GitLab](https://gitlab.com/pan93412), [Crowdin](https://crowdin.com/profile/pan93412) - Victor Wu - [GitLab](https://gitlab.com/_victorwu_), [Crowdin](https://crowdin.com/profile/victorwu) + - Hansel Wang - [GitLab](https://gitlab.com/airness), [Crowdin](https://crowdin.com/profile/airness) - Chinese Traditional, Hong Kong 繁體中文 (香港) - Victor Wu - [GitLab](https://gitlab.com/_victorwu_), [Crowdin](https://crowdin.com/profile/victorwu) - Ivan Ip - [GitLab](https://gitlab.com/lifehome), [Crowdin](https://crowdin.com/profile/lifehome) @@ -63,7 +64,6 @@ are very appreciative of the work done by translators and proofreaders! - German - Michael Hahnle - [GitLab](https://gitlab.com/mhah), [Crowdin](https://crowdin.com/profile/mhah) - Katrin Leinweber - [GitLab](https://gitlab.com/katrinleinweber), [Crowdin](https://crowdin.com/profile/katrinleinweber) - - Justman10000 - [GitLab](https://gitlab.com/Justman10000), [Crowdin](https://crowdin.com/profile/Justman10000) - Vladislav Wanner - [GitLab](https://gitlab.com/RumBugen), [Crowdin](https://crowdin.com/profile/RumBugen) - Greek - Proofreaders needed. diff --git a/doc/development/image_scaling.md b/doc/development/image_scaling.md index 93575429369..2078db8294c 100644 --- a/doc/development/image_scaling.md +++ b/doc/development/image_scaling.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Memory +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 2a29df380de..6cbbb6bf716 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -113,8 +113,8 @@ Marked stuck import jobs as failed. JIDs: xyz While the performance problems are not tackled, there is a process to workaround importing big projects, using a foreground import: -[Foreground import](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/5384) of big projects for customers. -(Using the import template in the [infrastructure tracker](https://gitlab.com/gitlab-com/gl-infra/infrastructure/)) +[Foreground import](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/5384) of big projects for customers. +(Using the import template in the [infrastructure tracker](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues)) ## Security diff --git a/doc/development/import_project.md b/doc/development/import_project.md index c63ba229921..7c55d2e2668 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -144,7 +144,7 @@ project files on disk. ##### Import is successful, but with a `Total number of not imported relations: XX` message, and issues are not created during the import If you receive a `Total number of not imported relations: XX` message, and issues -aren't created during the import, check [exceptions_json.log](../administration/logs.md#exceptions_jsonlog). +aren't created during the import, check [exceptions_json.log](../administration/logs/index.md#exceptions_jsonlog). You might see an error like `N is out of range for ActiveModel::Type::Integer with limit 4 bytes`, where `N` is the integer exceeding the 4-byte integer limit. If that's the case, you are likely hitting the issue with rebalancing of `relative_position` field of the issues. diff --git a/doc/development/index.md b/doc/development/index.md index 1b897db5097..34e6f466664 100644 --- a/doc/development/index.md +++ b/doc/development/index.md @@ -7,9 +7,9 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo description: "Development Guidelines: learn how to contribute to GitLab." --- -# Contributor and Development Docs +# Contribute to the development of GitLab -Learn the processes and technical information needed for contributing to GitLab. +Learn how to contribute to the development of the GitLab product. This content is intended for members of the GitLab Team as well as community contributors. Content specific to the GitLab Team should instead be included in diff --git a/doc/development/insert_into_tables_in_batches.md b/doc/development/insert_into_tables_in_batches.md index ebed3d16319..ced5332e880 100644 --- a/doc/development/insert_into_tables_in_batches.md +++ b/doc/development/insert_into_tables_in_batches.md @@ -1,196 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -description: "Sometimes it is necessary to store large amounts of records at once, which can be inefficient -when iterating collections and performing individual `save`s. With the arrival of `insert_all` -in Rails 6, which operates at the row level (that is, using `Hash`es), GitLab has added a set -of APIs that make it safe and simple to insert ActiveRecord objects in bulk." +redirect_to: 'database/insert_into_tables_in_batches.md' +remove_date: '2022-11-05' --- -# Insert into tables in batches +This document was moved to [another location](database/insert_into_tables_in_batches.md). -Sometimes it is necessary to store large amounts of records at once, which can be inefficient -when iterating collections and saving each record individually. With the arrival of -[`insert_all`](https://apidock.com/rails/ActiveRecord/Persistence/ClassMethods/insert_all) -in Rails 6, which operates at the row level (that is, using `Hash` objects), GitLab has added a set -of APIs that make it safe and simple to insert `ActiveRecord` objects in bulk. - -## Prepare `ApplicationRecord`s for bulk insertion - -In order for a model class to take advantage of the bulk insertion API, it has to include the -`BulkInsertSafe` concern first: - -```ruby -class MyModel < ApplicationRecord - # other includes here - # ... - include BulkInsertSafe # include this last - - # ... -end -``` - -The `BulkInsertSafe` concern has two functions: - -- It performs checks against your model class to ensure that it does not use ActiveRecord - APIs that are not safe to use with respect to bulk insertions (more on that below). -- It adds new class methods `bulk_insert!` and `bulk_upsert!`, which you can use to insert many records at once. - -## Insert records with `bulk_insert!` and `bulk_upsert!` - -If the target class passes the checks performed by `BulkInsertSafe`, you can insert an array of -ActiveRecord model objects as follows: - -```ruby -records = [MyModel.new, ...] - -MyModel.bulk_insert!(records) -``` - -Calls to `bulk_insert!` always attempt to insert _new records_. If instead -you would like to replace existing records with new values, while still inserting those -that do not already exist, then you can use `bulk_upsert!`: - -```ruby -records = [MyModel.new, existing_model, ...] - -MyModel.bulk_upsert!(records, unique_by: [:name]) -``` - -In this example, `unique_by` specifies the columns by which records are considered to be -unique and as such are updated if they existed prior to insertion. For example, if -`existing_model` has a `name` attribute, and if a record with the same `name` value already -exists, its fields are updated with those of `existing_model`. - -The `unique_by` parameter can also be passed as a `Symbol`, in which case it specifies -a database index by which a column is considered unique: - -```ruby -MyModel.bulk_insert!(records, unique_by: :index_on_name) -``` - -### Record validation - -The `bulk_insert!` method guarantees that `records` are inserted transactionally, and -runs validations on each record prior to insertion. If any record fails to validate, -an error is raised and the transaction is rolled back. You can turn off validations via -the `:validate` option: - -```ruby -MyModel.bulk_insert!(records, validate: false) -``` - -### Batch size configuration - -In those cases where the number of `records` is above a given threshold, insertions -occur in multiple batches. The default batch size is defined in -[`BulkInsertSafe::DEFAULT_BATCH_SIZE`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). -Assuming a default threshold of 500, inserting 950 records -would result in two batches being written sequentially (of size 500 and 450 respectively.) -You can override the default batch size via the `:batch_size` option: - -```ruby -MyModel.bulk_insert!(records, batch_size: 100) -``` - -Assuming the same number of 950 records, this would result in 10 batches being written instead. -Since this also affects the number of `INSERT` statements that occur, make sure you measure the -performance impact this might have on your code. There is a trade-off between the number of -`INSERT` statements the database has to process and the size and cost of each `INSERT`. - -### Handling duplicate records - -NOTE: -This parameter applies only to `bulk_insert!`. If you intend to update existing -records, use `bulk_upsert!` instead. - -It may happen that some records you are trying to insert already exist, which would result in -primary key conflicts. There are two ways to address this problem: failing fast by raising an -error or skipping duplicate records. The default behavior of `bulk_insert!` is to fail fast -and raise an `ActiveRecord::RecordNotUnique` error. - -If this is undesirable, you can instead skip duplicate records with the `skip_duplicates` flag: - -```ruby -MyModel.bulk_insert!(records, skip_duplicates: true) -``` - -### Requirements for safe bulk insertions - -Large parts of ActiveRecord's persistence API are built around the notion of callbacks. Many -of these callbacks fire in response to model life cycle events such as `save` or `create`. -These callbacks cannot be used with bulk insertions, since they are meant to be called for -every instance that is saved or created. Since these events do not fire when -records are inserted in bulk, we currently prevent their use. - -The specifics around which callbacks are explicitly allowed are defined in -[`BulkInsertSafe`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/concerns/bulk_insert_safe.rb). -Consult the module source code for details. If your class uses callbacks that are not explicitly designated -safe and you `include BulkInsertSafe` the application fails with an error. - -### `BulkInsertSafe` versus `InsertAll` - -Internally, `BulkInsertSafe` is based on `InsertAll`, and you may wonder when to choose -the former over the latter. To help you make the decision, -the key differences between these classes are listed in the table below. - -| | Input type | Validates input | Specify batch size | Can bypass callbacks | Transactional | -|--------------- | -------------------- | --------------- | ------------------ | --------------------------------- | ------------- | -| `bulk_insert!` | ActiveRecord objects | Yes (optional) | Yes (optional) | No (prevents unsafe callback use) | Yes | -| `insert_all!` | Attribute hashes | No | No | Yes | Yes | - -To summarize, `BulkInsertSafe` moves bulk inserts closer to how ActiveRecord objects -and inserts would normally behave. However, if all you need is to insert raw data in bulk, then -`insert_all` is more efficient. - -## Insert `has_many` associations in bulk - -A common use case is to save collections of associated relations through the owner side of the relation, -where the owned relation is associated to the owner through the `has_many` class method: - -```ruby -owner = OwnerModel.new(owned_relations: array_of_owned_relations) -# saves all `owned_relations` one-by-one -owner.save! -``` - -This issues a single `INSERT`, and transaction, for every record in `owned_relations`, which is inefficient if -`array_of_owned_relations` is large. To remedy this, the `BulkInsertableAssociations` concern can be -used to declare that the owner defines associations that are safe for bulk insertion: - -```ruby -class OwnerModel < ApplicationRecord - # other includes here - # ... - include BulkInsertableAssociations # include this last - - has_many :my_models -end -``` - -Here `my_models` must be declared `BulkInsertSafe` (as described previously) for bulk insertions -to happen. You can now insert any yet unsaved records as follows: - -```ruby -BulkInsertableAssociations.with_bulk_insert do - owner = OwnerModel.new(my_models: array_of_my_model_instances) - # saves `my_models` using a single bulk insert (possibly via multiple batches) - owner.save! -end -``` - -You can still save relations that are not `BulkInsertSafe` in this block; they -simply are treated as if you had invoked `save` from outside the block. - -## Known limitations - -There are a few restrictions to how these APIs can be used: - -- `BulkInsertableAssociations`: - - It is currently only compatible with `has_many` relations. - - It does not yet support `has_many through: ...` relations. - -Moreover, input data should either be limited to around 1000 records at most, -or already batched prior to calling bulk insert. The `INSERT` statement runs in a single -transaction, so for large amounts of records it may negatively affect database stability. +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index 5d1bd5ad61c..0387ba2e4dd 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -273,7 +273,7 @@ When developing a new integration, we also recommend you gate the availability b You can provide help text in the integration form, including links to off-site documentation, as described above in [Customize the frontend form](#customize-the-frontend-form). Refer to -our [usability guidelines](https://design.gitlab.com/usability/helping-users) for help text. +our [usability guidelines](https://design.gitlab.com/usability/helping-users/) for help text. For more detailed documentation, provide a page in `doc/user/project/integrations`, and link it from the [Integrations overview](../../user/project/integrations/index.md). diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 0a0c5e4d2a6..55e57a3c2ee 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -151,7 +151,7 @@ Depending on the CI infrastructure, the CI may have to fetch the Docker image every time the job runs. For the scanning job to run fast and avoid wasting bandwidth, Docker images should be as small as possible. You should aim for 50MB or smaller. If that isn't possible, try to keep it below 1.46 GB, -which is the size of a CD-ROM. +which is the size of a DVD-ROM. If the scanner requires a fully functional Linux environment, it is recommended to use a [Debian](https://www.debian.org/intro/about) "slim" distribution or [Alpine Linux](https://www.alpinelinux.org/). @@ -253,6 +253,10 @@ then `artifacts:reports:dependency_scanning` must be set to `depscan.json`. Following the POSIX exit code standard, the scanner exits with 0 for success and any number from 1 to 255 for anything else. Success also includes the case when vulnerabilities are found. +When a CI job fails, security report results are not ingested by GitLab, even if the job +[allows failure](../../ci/yaml/#allow_failure). The report artifacts are still uploaded to GitLab and available +for [download in the pipeline security tab](../../user/application_security/vulnerability_report/pipeline.md#download-security-scan-outputs). + When executing a scanning job using the [Docker-in-Docker privileged mode](../../user/application_security/sast/index.md#requirements), we reserve the following standard exit codes. @@ -310,7 +314,7 @@ This documentation gives an overview of the report JSON format, as well as recommendations and examples to help integrators set its fields. The format is extensively described in the documentation of [SAST](../../user/application_security/sast/index.md#reports-json-format), -[DAST](../../user/application_security/dast/#reports), +[DAST](../../user/application_security/dast/index.md#reports), [Dependency Scanning](../../user/application_security/dependency_scanning/index.md#reports-json-format), and [Container Scanning](../../user/application_security/container_scanning/index.md#reports-json-format) @@ -493,19 +497,20 @@ We recommend that you use the identifiers the GitLab scanners already define: |------------|------|---------------| | [CVE](https://cve.mitre.org/cve/) | `cve` | CVE-2019-10086 | | [CWE](https://cwe.mitre.org/data/index.html) | `cwe` | CWE-1026 | +| [ELSA](https://linux.oracle.com/security/) | `elsa` | ELSA-2020-0085 | | [OSVD](https://cve.mitre.org/data/refs/refmap/source-OSVDB.html) | `osvdb` | OSVDB-113928 | +| [OWASP](https://owasp.org/Top10/) | `owasp` | A01:2021–Broken Access Control Design | +| [RHSA](https://access.redhat.com/errata-search/#/) | `rhsa` | RHSA-2020:0111 | | [USN](https://ubuntu.com/security/notices) | `usn` | USN-4234-1 | | [WASC](http://projects.webappsec.org/Threat-Classification-Reference-Grid) | `wasc` | WASC-19 | -| [RHSA](https://access.redhat.com/errata/#/) | `rhsa` | RHSA-2020:0111 | -| [ELSA](https://linux.oracle.com/security/) | `elsa` | ELSA-2020-0085 | The generic identifiers listed above are defined in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common), which is shared by some of the analyzers that GitLab maintains. You can [contribute](https://gitlab.com/gitlab-org/security-products/analyzers/common/blob/master/issue/identifier.go) new generic identifiers to if needed. Analyzers may also produce vendor-specific or product-specific identifiers, which don't belong in the [common library](https://gitlab.com/gitlab-org/security-products/analyzers/common). -The first item of the `identifiers` array is called the [primary -identifier](../../user/application_security/terminology/#primary-identifier). +The first item of the `identifiers` array is called the +[primary identifier](../../user/application_security/terminology/index.md#primary-identifier). The primary identifier is particularly important, because it is used to [track vulnerabilities](#tracking-and-merging-vulnerabilities) as new commits are pushed to the repository. Identifiers are also used to [merge duplicate vulnerabilities](#tracking-and-merging-vulnerabilities) diff --git a/doc/development/internal_api/index.md b/doc/development/internal_api/index.md index 13e095b4a83..9b29af3e433 100644 --- a/doc/development/internal_api/index.md +++ b/doc/development/internal_api/index.md @@ -148,8 +148,8 @@ curl --request POST --header "Gitlab-Shared-Secret: <Base64 encoded token>" \ ## Authorized Keys Check This endpoint is called by the GitLab Shell authorized keys -check. Which is called by OpenSSH for [fast SSH key -lookup](../../administration/operations/fast_ssh_key_lookup.md). +check. Which is called by OpenSSH for +[fast SSH key lookup](../../administration/operations/fast_ssh_key_lookup.md). | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| @@ -494,10 +494,15 @@ curl --request GET --header "Gitlab-Kas-Api-Request: <JWT token>" \ Called from GitLab agent server (`kas`) to increase the usage metric counters. -| Attribute | Type | Required | Description | -|:----------|:-------|:---------|:------------| -| `gitops_sync_count` | integer| no | The number to increase the `gitops_sync_count` counter by | -| `k8s_api_proxy_request_count` | integer| no | The number to increase the `k8s_api_proxy_request_count` counter by | +| Attribute | Type | Required | Description | +|:---------------------------------------------------------------------------|:--------------|:---------|:-----------------------------------------------------------------------------------------------------------------| +| `gitops_sync_count` (DEPRECATED) | integer | no | The number to increase the `gitops_sync` counter by | +| `k8s_api_proxy_request_count` (DEPRECATED) | integer | no | The number to increase the `k8s_api_proxy_request` counter by | +| `counters` | hash | no | The number to increase the `k8s_api_proxy_request` counter by | +| `counters["k8s_api_proxy_request"]` | integer | no | The number to increase the `k8s_api_proxy_request` counter by | +| `counters["gitops_sync"]` | integer | no | The number to increase the `gitops_sync` counter by | +| `unique_counters` | hash | no | The number to increase the `k8s_api_proxy_request` counter by | +| `unique_counters["agent_users_using_ci_tunnel"]` | integer array | no | The set of unique user ids that have interacted a CI Tunnel to track the `agent_users_using_ci_tunnel` metric event | ```plaintext POST /internal/kubernetes/usage_metrics @@ -624,6 +629,40 @@ Example response: } ``` +### Policy Configuration + +Called from GitLab agent server (`kas`) to retrieve `policies_configuration` +configured for the project belonging to the agent token. GitLab `kas` uses +this to configure the agent to scan images in the Kubernetes cluster based on the configuration. + +```plaintext +GET /internal/kubernetes/modules/starboard_vulnerability/policies_configuration +``` + +Example Request: + +```shell +curl --request GET --header "Gitlab-Kas-Api-Request: <JWT token>" \ + --header "Authorization: Bearer <agent token>" "http://localhost:3000/api/v4/internal/kubernetes/modules/starboard_vulnerability/policies_configuration" +``` + +Example response: + +```json +{ + "configurations": [ + { + "cadence": "30 2 * * *", + "namespaces": [ + "namespace-a", + "namespace-b" + ], + "updated_at": "2022-06-02T05:36:26+00:00" + } + ] +} +``` + ## Subscriptions The subscriptions endpoint is used by [CustomersDot](https://gitlab.com/gitlab-org/customers-gitlab-com) (`customers.gitlab.com`) diff --git a/doc/development/issue_types.md b/doc/development/issue_types.md index e6047c62827..820c37aeb14 100644 --- a/doc/development/issue_types.md +++ b/doc/development/issue_types.md @@ -13,10 +13,8 @@ Sometimes when a new resource type is added it's not clear if it should be only "extension" of Issue (Issue Type) or if it should be a new first-class resource type (similar to issue, epic, merge request, snippet). -The idea of Issue Types was first proposed in [this -issue](https://gitlab.com/gitlab-org/gitlab/-/issues/8767) and its usage was -discussed few times since then, for example in [incident -management](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55532). +The idea of Issue Types was first proposed in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/8767) and its usage was +discussed few times since then, for example in [incident management](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/55532). ## What is an Issue Type diff --git a/doc/development/iterating_tables_in_batches.md b/doc/development/iterating_tables_in_batches.md index 1159e3755e5..589e38a5cb0 100644 --- a/doc/development/iterating_tables_in_batches.md +++ b/doc/development/iterating_tables_in_batches.md @@ -1,598 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/iterating_tables_in_batches.md' +remove_date: '2022-11-06' --- -# Iterating tables in batches +This document was moved to [another location](database/iterating_tables_in_batches.md). -Rails provides a method called `in_batches` that can be used to iterate over -rows in batches. For example: - -```ruby -User.in_batches(of: 10) do |relation| - relation.update_all(updated_at: Time.now) -end -``` - -Unfortunately, this method is implemented in a way that is not very efficient, -both query and memory usage wise. - -To work around this you can include the `EachBatch` module into your models, -then use the `each_batch` class method. For example: - -```ruby -class User < ActiveRecord::Base - include EachBatch -end - -User.each_batch(of: 10) do |relation| - relation.update_all(updated_at: Time.now) -end -``` - -This produces queries such as: - -```plaintext -User Load (0.7ms) SELECT "users"."id" FROM "users" WHERE ("users"."id" >= 41654) ORDER BY "users"."id" ASC LIMIT 1 OFFSET 1000 - (0.7ms) SELECT COUNT(*) FROM "users" WHERE ("users"."id" >= 41654) AND ("users"."id" < 42687) -``` - -The API of this method is similar to `in_batches`, though it doesn't support -all of the arguments that `in_batches` supports. You should always use -`each_batch` _unless_ you have a specific need for `in_batches`. - -## Iterating over non-unique columns - -One should proceed with extra caution. When you iterate over an attribute that is not unique, -even with the applied max batch size, there is no guarantee that the resulting batches do not -surpass it. The following snippet demonstrates this situation when one attempt to select -`Ci::Build` entries for users with `id` between `1` and `10,000`, the database returns -`1 215 178` matching rows. - -```ruby -[ gstg ] production> Ci::Build.where(user_id: (1..10_000)).size -=> 1215178 -``` - -This happens because the built relation is translated into the following query: - -```ruby -[ gstg ] production> puts Ci::Build.where(user_id: (1..10_000)).to_sql -SELECT "ci_builds".* FROM "ci_builds" WHERE "ci_builds"."type" = 'Ci::Build' AND "ci_builds"."user_id" BETWEEN 1 AND 10000 -=> nil -``` - -`And` queries which filter non-unique column by range `WHERE "ci_builds"."user_id" BETWEEN ? AND ?`, -even though the range size is limited to a certain threshold (`10,000` in the previous example) this -threshold does not translate to the size of the returned dataset. That happens because when taking -`n` possible values of attributes, one can't tell for sure that the number of records that contains -them is less than `n`. - -### Loose-index scan with `distinct_each_batch` - -When iterating over a non-unique column is necessary, use the `distinct_each_batch` helper -method. The helper uses the [loose-index scan technique](https://wiki.postgresql.org/wiki/Loose_indexscan) -(skip-index scan) to skip duplicated values within a database index. - -Example: iterating over distinct `author_id` in the Issue model - -```ruby -Issue.distinct_each_batch(column: :author_id, of: 1000) do |relation| - users = User.where(id: relation.select(:author_id)).to_a -end -``` - -The technique provides stable performance between the batches regardless of the data distribution. -The `relation` object returns an ActiveRecord scope where only the given `column` is available. -Other columns are not loaded. - -The underlying database queries use recursive CTEs, which adds extra overhead. We therefore advise to use -smaller batch sizes than those used for a standard `each_batch` iteration. - -## Column definition - -`EachBatch` uses the primary key of the model by default for the iteration. This works most of the -cases, however in some cases, you might want to use a different column for the iteration. - -```ruby -Project.distinct.each_batch(column: :creator_id, of: 10) do |relation| - puts User.where(id: relation.select(:creator_id)).map(&:id) -end -``` - -The query above iterates over the project creators and prints them out without duplications. - -NOTE: -In case the column is not unique (no unique index definition), calling the `distinct` method on -the relation is necessary. Using not unique column without `distinct` may result in `each_batch` -falling into an endless loop as described in following -[issue](https://gitlab.com/gitlab-org/gitlab/-/issues/285097). - -## `EachBatch` in data migrations - -When dealing with data migrations the preferred way to iterate over a large volume of data is using -`EachBatch`. - -A special case of data migration is a [background migration](database/background_migrations.md#scheduling) -where the actual data modification is executed in a background job. The migration code that -determines the data ranges (slices) and schedules the background jobs uses `each_batch`. - -## Efficient usage of `each_batch` - -`EachBatch` helps to iterate over large tables. It's important to highlight that `EachBatch` -does not magically solve all iteration-related performance problems, and it might not help at -all in some scenarios. From the database point of view, correctly configured database indexes are -also necessary to make `EachBatch` perform well. - -### Example 1: Simple iteration - -Let's consider that we want to iterate over the `users` table and print the `User` records to the -standard output. The `users` table contains millions of records, thus running one query to fetch -the users likely times out. - - - -This is a simplified version of the `users` table which contains several rows. We have a few -smaller gaps in the `id` column to make the example a bit more realistic (a few records were -already deleted). Currently, we have one index on the `id` field. - -Loading all users into memory (avoid): - -```ruby -users = User.all - -users.each { |user| puts user.inspect } -``` - -Use `each_batch`: - -```ruby -# Note: for this example I picked 5 as the batch size, the default is 1_000 -User.each_batch(of: 5) do |relation| - relation.each { |user| puts user.inspect } -end -``` - -#### How `each_batch` works - -As the first step, it finds the lowest `id` (start `id`) in the table by executing the following -database query: - -```sql -SELECT "users"."id" FROM "users" ORDER BY "users"."id" ASC LIMIT 1 -``` - - - -Notice that the query only reads data from the index (`INDEX ONLY SCAN`), the table is not -accessed. Database indexes are sorted so taking out the first item is a very cheap operation. - -The next step is to find the next `id` (end `id`) which should respect the batch size -configuration. In this example we used a batch size of 5. `EachBatch` uses the `OFFSET` clause -to get a "shifted" `id` value. - -```sql -SELECT "users"."id" FROM "users" WHERE "users"."id" >= 1 ORDER BY "users"."id" ASC LIMIT 1 OFFSET 5 -``` - - - -Again, the query only looks into the index. The `OFFSET 5` takes out the sixth `id` value: this -query reads a maximum of six items from the index regardless of the table size or the iteration -count. - -At this point, we know the `id` range for the first batch. Now it's time to construct the query -for the `relation` block. - -```sql -SELECT "users".* FROM "users" WHERE "users"."id" >= 1 AND "users"."id" < 302 -``` - - - -Notice the `<` sign. Previously six items were read from the index and in this query, the last -value is "excluded". The query looks at the index to get the location of the five `user` -rows on the disk and read the rows from the table. The returned array is processed in Ruby. - -The first iteration is done. For the next iteration, the last `id` value is reused from the -previous iteration in order to find out the next end `id` value. - -```sql -SELECT "users"."id" FROM "users" WHERE "users"."id" >= 302 ORDER BY "users"."id" ASC LIMIT 1 OFFSET 5 -``` - - - -Now we can easily construct the `users` query for the second iteration. - -```sql -SELECT "users".* FROM "users" WHERE "users"."id" >= 302 AND "users"."id" < 353 -``` - - - -### Example 2: Iteration with filters - -Building on top of the previous example, we want to print users with zero sign-in count. We keep -track of the number of sign-ins in the `sign_in_count` column so we write the following code: - -```ruby -users = User.where(sign_in_count: 0) - -users.each_batch(of: 5) do |relation| - relation.each { |user| puts user.inspect } -end -``` - -`each_batch` produces the following SQL query for the start `id` value: - -```sql -SELECT "users"."id" FROM "users" WHERE "users"."sign_in_count" = 0 ORDER BY "users"."id" ASC LIMIT 1 -``` - -Selecting only the `id` column and ordering by `id` forces the database to use the -index on the `id` (primary key index) column however, we also have an extra condition on the -`sign_in_count` column. The column is not part of the index, so the database needs to look into -the actual table to find the first matching row. - - - -NOTE: -The number of scanned rows depends on the data distribution in the table. - -- Best case scenario: the first user was never logged in. The database reads only one row. -- Worst case scenario: all users were logged in at least once. The database reads all rows. - -In this particular example, the database had to read 10 rows (regardless of our batch size setting) -to determine the first `id` value. In a "real-world" application it's hard to predict whether the -filtering causes problems or not. In the case of GitLab, verifying the data on a -production replica is a good start, but keep in mind that data distribution on GitLab.com can be -different from self-managed instances. - -#### Improve filtering with `each_batch` - -##### Specialized conditional index - -```sql -CREATE INDEX index_on_users_never_logged_in ON users (id) WHERE sign_in_count = 0 -``` - -This is how our table and the newly created index looks like: - - - -This index definition covers the conditions on the `id` and `sign_in_count` columns thus makes the -`each_batch` queries very effective (similar to the simple iteration example). - -It's rare when a user was never signed in so we a anticipate small index size. Including only the -`id` in the index definition also helps to keep the index size small. - -##### Index on columns - -Later on, we might want to iterate over the table filtering for different `sign_in_count` values, in -those cases we cannot use the previously suggested conditional index because the `WHERE` condition -does not match with our new filter (`sign_in_count > 10`). - -To address this problem, we have two options: - -- Create another, conditional index to cover the new query. -- Replace the index with a more generalized configuration. - -NOTE: -Having multiple indexes on the same table and on the same columns could be a performance bottleneck -when writing data. - -Let's consider the following index (avoid): - -```sql -CREATE INDEX index_on_users_never_logged_in ON users (id, sign_in_count) -``` - -The index definition starts with the `id` column which makes the index very inefficient from data -selectivity point of view. - -```sql -SELECT "users"."id" FROM "users" WHERE "users"."sign_in_count" = 0 ORDER BY "users"."id" ASC LIMIT 1 -``` - -Executing the query above results in an `INDEX ONLY SCAN`. However, the query still needs to -iterate over an unknown number of entries in the index, and then find the first item where the -`sign_in_count` is `0`. - - - -We can improve the query significantly by swapping the columns in the index definition (prefer). - -```sql -CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count, id) -``` - - - -The following index definition does not work well with `each_batch` (avoid). - -```sql -CREATE INDEX index_on_users_never_logged_in ON users (sign_in_count) -``` - -Since `each_batch` builds range queries based on the `id` column, this index cannot be used -efficiently. The DB reads the rows from the table or uses a bitmap search where the primary -key index is also read. - -##### "Slow" iteration - -Slow iteration means that we use a good index configuration to iterate over the table and -apply filtering on the yielded relation. - -```ruby -User.each_batch(of: 5) do |relation| - relation.where(sign_in_count: 0).each { |user| puts user inspect } -end -``` - -The iteration uses the primary key index (on the `id` column) which makes it safe from statement -timeouts. The filter (`sign_in_count: 0`) is applied on the `relation` where the `id` is already -constrained (range). The number of rows is limited. - -Slow iteration generally takes more time to finish. The iteration count is higher and -one iteration could yield fewer records than the batch size. Iterations may even yield -0 records. This is not an optimal solution; however, in some cases (especially when -dealing with large tables) this is the only viable option. - -### Using Subqueries - -Using subqueries in your `each_batch` query does not work well in most cases. Consider the following example: - -```ruby -projects = Project.where(creator_id: Issue.where(confidential: true).select(:author_id)) - -projects.each_batch do |relation| - # do something -end -``` - -The iteration uses the `id` column of the `projects` table. The batching does not affect the -subquery. This means for each iteration, the subquery is executed by the database. This adds a -constant "load" on the query which often ends up in statement timeouts. We have an unknown number -of [confidential issues](../user/project/issues/confidential_issues.md), the execution time -and the accessed database rows depend on the data distribution in the `issues` table. - -NOTE: -Using subqueries works only when the subquery returns a small number of rows. - -#### Improving Subqueries - -When dealing with subqueries, a slow iteration approach could work: the filter on `creator_id` -can be part of the generated `relation` object. - -```ruby -projects = Project.all - -projects.each_batch do |relation| - relation.where(creator_id: Issue.where(confidential: true).select(:author_id)) -end -``` - -If the query on the `issues` table itself is not performant enough, a nested loop could be -constructed. Try to avoid it when possible. - -```ruby -projects = Project.all - -projects.each_batch do |relation| - issues = Issue.where(confidential: true) - - issues.each_batch do |issues_relation| - relation.where(creator_id: issues_relation.select(:author_id)) - end -end -``` - -If we know that the `issues` table has many more rows than `projects`, it would make sense to flip -the queries, where the `issues` table is batched first. - -### Using `JOIN` and `EXISTS` - -When to use `JOINS`: - -- When there's a 1:1 or 1:N relationship between the tables where we know that the joined record -(almost) always exists. This works well for "extension-like" tables: - - `projects` - `project_settings` - - `users` - `user_details` - - `users` - `user_statuses` -- `LEFT JOIN` works well in this case. Conditions on the joined table need to go to the yielded -relation so the iteration is not affected by the data distribution in the joined table. - -Example: - -```ruby -users = User.joins("LEFT JOIN personal_access_tokens on personal_access_tokens.user_id = users.id") - -users.each_batch do |relation| - relation.where("personal_access_tokens.name = 'name'") -end -``` - -`EXISTS` queries should be added only to the inner `relation` of the `each_batch` query: - -```ruby -User.each_batch do |relation| - relation.where("EXISTS (SELECT 1 FROM ...") -end -``` - -### Complex queries on the relation object - -When the `relation` object has several extra conditions, the execution plans might become -"unstable". - -Example: - -```ruby -Issue.each_batch do |relation| - relation - .joins(:metrics) - .joins(:merge_requests_closing_issues) - .where("id IN (SELECT ...)") - .where(confidential: true) -end -``` - -Here, we expect that the `relation` query reads the `BATCH_SIZE` of user records and then -filters down the results according to the provided queries. The planner might decide that -using a bitmap index lookup with the index on the `confidential` column is a better way to -execute the query. This can cause an unexpectedly high amount of rows to be read and the -query could time out. - -Problem: we know for sure that the relation is returning maximum `BATCH_SIZE` of records -however, the planner does not know this. - -Common table expression (CTE) trick to force the range query to execute first: - -```ruby -Issue.each_batch(of: 1000) do |relation| - cte = Gitlab::SQL::CTE.new(:batched_relation, relation.limit(1000)) - - scope = cte - .apply_to(Issue.all) - .joins(:metrics) - .joins(:merge_requests_closing_issues) - .where("id IN (SELECT ...)") - .where(confidential: true) - - puts scope.to_a -end -``` - -### `EachBatch` vs `BatchCount` - -When adding new counters for Service Ping, the preferred way to count records is using the -`Gitlab::Database::BatchCount` class. The iteration logic implemented in `BatchCount` -has similar performance characteristics like `EachBatch`. Most of the tips and suggestions -for improving `BatchCount` mentioned above applies to `BatchCount` as well. - -## Iterate with keyset pagination - -There are a few special cases where iterating with `EachBatch` does not work. `EachBatch` -requires one distinct column (usually the primary key), which makes the iteration impossible -for timestamp columns and tables with composite primary keys. - -Where `EachBatch` does not work, you can use -[keyset pagination](database/pagination_guidelines.md#keyset-pagination) to iterate over the -table or a range of rows. The scaling and performance characteristics are very similar to -`EachBatch`. - -Examples: - -- Iterate over the table in a specific order (timestamp columns) in combination with a tie-breaker -if column user to sort by does not contain unique values. -- Iterate over the table with composite primary keys. - -### Iterate over the issues in a project by creation date - -You can use keyset pagination to iterate over any database column in a specific order (for example, -`created_at DESC`). To ensure consistent order of the returned records with the same values for -`created_at`, use a tie-breaker column with unique values (for example, `id`). - -Assume you have the following index in the `issues` table: - -```sql -idx_issues_on_project_id_and_created_at_and_id" btree (project_id, created_at, id) -``` - -### Fetching records for further processing - -The following snippet iterates over issue records within the project using the specified order -(`created_at, id`). - -```ruby -scope = Issue.where(project_id: 278964).order(:created_at, :id) # id is the tie-breaker - -iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) - -iterator.each_batch(of: 100) do |records| - puts records.map(&:id) -end -``` - -You can add extra filters to the query. This example only lists the issue IDs created in the last -30 days: - -```ruby -scope = Issue.where(project_id: 278964).where('created_at > ?', 30.days.ago).order(:created_at, :id) # id is the tie-breaker - -iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) - -iterator.each_batch(of: 100) do |records| - puts records.map(&:id) -end -``` - -### Updating records in the batch - -For complex `ActiveRecord` queries, the `.update_all` method does not work well, because it -generates an incorrect `UPDATE` statement. -You can use raw SQL for updating records in batches: - -```ruby -scope = Issue.where(project_id: 278964).order(:created_at, :id) # id is the tie-breaker - -iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) - -iterator.each_batch(of: 100) do |records| - ApplicationRecord.connection.execute("UPDATE issues SET updated_at=NOW() WHERE issues.id in (#{records.dup.reselect(:id).to_sql})") -end -``` - -NOTE: -To keep the iteration stable and predictable, avoid updating the columns in the `ORDER BY` clause. - -### Iterate over the `merge_request_diff_commits` table - -The `merge_request_diff_commits` table uses a composite primary key (`merge_request_diff_id, -relative_order`), which makes `EachBatch` impossible to use efficiently. - -To paginate over the `merge_request_diff_commits` table, you can use the following snippet: - -```ruby -# Custom order object configuration: -order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'merge_request_diff_id', - order_expression: MergeRequestDiffCommit.arel_table[:merge_request_diff_id].asc, - nullable: :not_nullable, - distinct: false, - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: 'relative_order', - order_expression: MergeRequestDiffCommit.arel_table[:relative_order].asc, - nullable: :not_nullable, - distinct: false, - ) -]) -MergeRequestDiffCommit.include(FromUnion) # keyset pagination generates UNION queries - -scope = MergeRequestDiffCommit.order(order) - -iterator = Gitlab::Pagination::Keyset::Iterator.new(scope: scope) - -iterator.each_batch(of: 100) do |records| - puts records.map { |record| [record.merge_request_diff_id, record.relative_order] }.inspect -end -``` - -### Order object configuration - -Keyset pagination works well with simple `ActiveRecord` `order` scopes -([first example](iterating_tables_in_batches.md#iterate-over-the-issues-in-a-project-by-creation-date). -However, in special cases, you need to describe the columns in the `ORDER BY` clause (second example) -for the underlying keyset pagination library. When the `ORDER BY` configuration cannot be -automatically determined by the keyset pagination library, an error is raised. - -The code comments of the -[`Gitlab::Pagination::Keyset::Order`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/pagination/keyset/order.rb) -and [`Gitlab::Pagination::Keyset::ColumnOrderDefinition`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/pagination/keyset/column_order_definition.rb) -classes give an overview of the possible options for configuring the `ORDER BY` clause. You can -also find a few code examples in the -[keyset pagination](database/keyset_pagination.md#complex-order-configuration) documentation. +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/jh_features_review.md b/doc/development/jh_features_review.md index 88830a80bf1..7b81ecfe8f5 100644 --- a/doc/development/jh_features_review.md +++ b/doc/development/jh_features_review.md @@ -62,7 +62,8 @@ For features that build on existing CE/EE features, a module in the `JH` namespace injected in the CE/EE class/module is needed. This aligns with what we're doing with EE features. -See [EE features based on CE features](ee_features.md#ee-features-based-on-ce-features) for more details. +See [Extend CE features with EE backend code](ee_features.md#extend-ce-features-with-ee-backend-code) +for more details. For example, to prepend a module into the `User` class you would use the following approach: diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index ee261769d82..1c83bef5620 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -155,7 +155,7 @@ Mitigation strategies include: ## Debugging Kubernetes integrations Logs related to the Kubernetes integration can be found in -[`kubernetes.log`](../administration/logs.md#kuberneteslog). On a local +[`kubernetes.log`](../administration/logs/index.md#kuberneteslog). On a local GDK install, these logs are present in `log/kubernetes.log`. Some services such as diff --git a/doc/development/lfs.md b/doc/development/lfs.md index 9b78c8869b1..5900eb68294 100644 --- a/doc/development/lfs.md +++ b/doc/development/lfs.md @@ -76,14 +76,13 @@ process, which writes the contents to the standard output. 1. The archive data is sent back to the client. In step 7, the `gitaly-lfs-smudge` filter must talk to Workhorse, not to -Rails, or an invalid LFS blob is saved. To support this, GitLab -13.5 [changed the default Omnibus configuration to have Gitaly talk to -the Workhorse](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4592) +Rails, or an invalid LFS blob is saved. To support this, GitLab 13.5 +[changed the default Omnibus configuration to have Gitaly talk to the Workhorse](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4592) instead of Rails. One side effect of this change: the correlation ID of the original request is not preserved for the internal API requests made by Gitaly (or `gitaly-lfs-smudge`), such as the one made in step 8. The -correlation IDs for those API requests are random values until [this -Workhorse issue](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/309) is +correlation IDs for those API requests are random values until +[this Workhorse issue](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/309) is resolved. diff --git a/doc/development/licensed_feature_availability.md b/doc/development/licensed_feature_availability.md index 21b07ae89b5..b007df8f1da 100644 --- a/doc/development/licensed_feature_availability.md +++ b/doc/development/licensed_feature_availability.md @@ -1,72 +1,11 @@ --- -stage: Fulfillment -group: Provision -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'ee_features.md' +remove_date: '2022-10-08' --- -# Licensed feature availability +This document was moved to [another location](ee_features.md). -As of GitLab 9.4, we've been supporting a simplified version of licensed -feature availability checks via `ee/app/models/license.rb`, both for -on-premise or GitLab.com plans and features. - -## Restricting features scoped by namespaces or projects - -GitLab.com plans are persisted on user groups and namespaces, therefore, if you're adding a -feature such as [Related issues](../user/project/issues/related_issues.md) or -[Service Desk](../user/project/service_desk.md), -it should be restricted on namespace scope. - -1. Add the feature symbol on `STARTER_FEATURES`, `PREMIUM_FEATURES`, or `ULTIMATE_FEATURES` constants in - `ee/app/models/gitlab_subscriptions/features.rb`. -1. Check using: - -```ruby -project.licensed_feature_available?(:feature_symbol) -``` - -or - -```ruby -group.licensed_feature_available?(:feature_symbol) -``` - -For projects, `licensed_feature_available` delegates to its associated `namespace`. - -## Restricting global features (instance) - -However, for features such as [Geo](../administration/geo/index.md) and -[Database Load Balancing](../administration/postgresql/database_load_balancing.md), which cannot be restricted -to only a subset of projects or namespaces, the check is made directly in -the instance license. - -1. Add the feature symbol to `STARTER_FEATURES`, `PREMIUM_FEATURES` or `ULTIMATE_FEATURES` constants in - `ee/app/models/gitlab_subscriptions/features.rb`. -1. Add the same feature symbol to `GLOBAL_FEATURES`. -1. Check using: - -```ruby -License.feature_available?(:feature_symbol) -``` - -## Restricting frontend features - -To restrict frontend features based on the license, use `push_licensed_feature`. -The frontend can then access this via `this.glFeatures`: - -```ruby -before_action do - push_licensed_feature(:feature_symbol) - # or by project/namespace - push_licensed_feature(:feature_symbol, project) -end -``` - -## Allow use of licensed EE features - -To enable plans per namespace turn on the `Allow use of licensed EE features` option from the settings page. -This will make licensed EE features available to projects only if the project namespace's plan includes the feature -or if the project is public. To enable it: - -1. If you are developing locally, follow the steps in [simulate SaaS](ee_features.md#act-as-saas) to make the option available. -1. Select Admin > Settings > General > "Account and limit" and enable "Allow use of licensed EE features". +<!-- This redirect file can be deleted after <2022-10-08>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/logging.md b/doc/development/logging.md index 749f85c9e2d..f1fa7f4c8c9 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Developers Guide to Logging **(FREE)** -[GitLab Logs](../administration/logs.md) play a critical role for both +[GitLab Logs](../administration/logs/index.md) play a critical role for both administrators and GitLab team members to diagnose problems in the field. ## Don't use `Rails.logger` @@ -44,8 +44,7 @@ These logs suffer from a number of problems: Note that currently on GitLab.com, any messages in `production.log` aren't indexed by Elasticsearch due to the sheer volume and noise. They do end up in Google Stackdriver, but it is still harder to search for -logs there. See the [GitLab.com logging -documentation](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/logging) +logs there. See the [GitLab.com logging documentation](https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/logging) for more details. ## Use structured (JSON) logging @@ -66,7 +65,7 @@ Suppose you want to log the events that happen in a project importer. You want to log issues created, merge requests, and so on, as the importer progresses. Here's what to do: -1. Look at [the list of GitLab Logs](../administration/logs.md) to see +1. Look at [the list of GitLab Logs](../administration/logs/index.md) to see if your log message might belong with one of the existing log files. 1. If there isn't a good place, consider creating a new filename, but check with a maintainer if it makes sense to do so. A log file should @@ -386,18 +385,18 @@ end ## Additional steps with new log files 1. Consider log retention settings. By default, Omnibus rotates any - logs in `/var/log/gitlab/gitlab-rails/*.log` every hour and [keep at - most 30 compressed files](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate). + logs in `/var/log/gitlab/gitlab-rails/*.log` every hour and + [keep at most 30 compressed files](https://docs.gitlab.com/omnibus/settings/logs.html#logrotate). On GitLab.com, that setting is only 6 compressed files. These settings should suffice for most users, but you may need to tweak them in [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab). -1. If you add a new file, submit an issue to the [production - tracker](https://gitlab.com/gitlab-com/gl-infra/production/-/issues) or +1. If you add a new file, submit an issue to the + [production tracker](https://gitlab.com/gitlab-com/gl-infra/production/-/issues) or a merge request to the [`gitlab_fluentd`](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd) project. See [this example](https://gitlab.com/gitlab-cookbooks/gitlab_fluentd/-/merge_requests/51/diffs). -1. Be sure to update the [GitLab CE/EE documentation](../administration/logs.md) and the [GitLab.com - runbooks](https://gitlab.com/gitlab-com/runbooks/blob/master/docs/logging/README.md). +1. Be sure to update the [GitLab CE/EE documentation](../administration/logs/index.md) and the + [GitLab.com runbooks](https://gitlab.com/gitlab-com/runbooks/blob/master/docs/logging/README.md). ## Control logging visibility diff --git a/doc/development/merge_request_concepts/index.md b/doc/development/merge_request_concepts/index.md index 8df0da5123e..331f0e01579 100644 --- a/doc/development/merge_request_concepts/index.md +++ b/doc/development/merge_request_concepts/index.md @@ -5,13 +5,11 @@ group: Code Review info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments-to-development-guidelines" --- -# Merge Request Concepts +# Merge request concepts -**NOTE**: +NOTE: The documentation below is the single source of truth for the merge request terminology and functionality. -## Overview - The merge request is made up of several different key components and ideas that encompass the overall merge request experience. These concepts sometimes have competing and confusing terminology or overlap with other concepts. The concepts this will cover are: 1. Merge widget @@ -19,7 +17,12 @@ The merge request is made up of several different key components and ideas that 1. Merge checks 1. Approval rules -### Merge widget +When developing new merge request widgets, read the +[merge request widget extension framework](../fe_guide/merge_request_widget_extensions.md) +documentation. All new widgets should use this framework, and older widgets should +be ported to use it. + +## Merge widget The merge widget is the component of the merge request where the `merge` button exists: @@ -27,27 +30,27 @@ The merge widget is the component of the merge request where the `merge` button This area of the merge request is where all of the options and commit messages are defined prior to merging. It also contains information about what is in the merge request, what issues may be closed, and other important information to the merging process. -### Report widgets +## Report widgets Reports are widgets within the merge request that report information about changes within the merge request. These widgets provide information to better help the author understand the changes and further improvements to the proposed changes. -[Design Documentation](https://design.gitlab.com/regions/merge-request-reports) +[Design Documentation](https://design.gitlab.com/regions/merge-request-reports/)  -### Merge checks +## Merge checks Merge checks are statuses that can either pass or fail and conditionally control the availability of the merge button being available within a merge request. The key distinguishing factor in a merge check is that users **do not** interact with the merge checks inside of the merge request, but are able to influence whether or not the check passes or fails. Results from the check are processed as true/false to determine whether or not a merge request can be merged. Examples include: -1. merge conflicts -1. pipeline success -1. threads resolution -1. [external status checks](../../user/project/merge_requests/status_checks.md) -1. required approvals +- Merge conflicts. +- Pipeline success. +- Threads resolution. +- [External status checks](../../user/project/merge_requests/status_checks.md). +- Required approvals. When all of the required merge checks are satisfied a merge request becomes mergeable. -### Approvals +## Approvals Approval rules specify users that are required to or can optionally approve a merge request based on some kind of organizational policy. When approvals are required, they effectively become a required merge check. The key differentiator between merge checks and approval rules is that users **do** interact with approval rules, by deciding to approve the merge request. diff --git a/doc/development/merge_request_concepts/widget_extensions.md b/doc/development/merge_request_concepts/widget_extensions.md new file mode 100644 index 00000000000..097e9155f2b --- /dev/null +++ b/doc/development/merge_request_concepts/widget_extensions.md @@ -0,0 +1,11 @@ +--- +redirect_to: '../fe_guide/merge_request_widget_extensions.md' +remove_date: '2022-10-27' +--- + +This document was moved to [another location](../fe_guide/merge_request_widget_extensions.md). + +<!-- This redirect file can be deleted after <2022-10-27>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md index 5e7fe9cc8fb..7ff25705ae6 100644 --- a/doc/development/merge_request_performance_guidelines.md +++ b/doc/development/merge_request_performance_guidelines.md @@ -193,7 +193,7 @@ costly, time-consuming query to the replicas. ## Use CTEs wisely -Read about [complex queries on the relation object](iterating_tables_in_batches.md#complex-queries-on-the-relation-object) for considerations on how to use CTEs. We have found in some situations that CTEs can become problematic in use (similar to the n+1 problem above). In particular, hierarchical recursive CTE queries such as the CTE in [AuthorizedProjectsWorker](https://gitlab.com/gitlab-org/gitlab/-/issues/325688) are very difficult to optimize and don't scale. We should avoid them when implementing new features that require any kind of hierarchical structure. +Read about [complex queries on the relation object](database/iterating_tables_in_batches.md#complex-queries-on-the-relation-object) for considerations on how to use CTEs. We have found in some situations that CTEs can become problematic in use (similar to the n+1 problem above). In particular, hierarchical recursive CTE queries such as the CTE in [AuthorizedProjectsWorker](https://gitlab.com/gitlab-org/gitlab/-/issues/325688) are very difficult to optimize and don't scale. We should avoid them when implementing new features that require any kind of hierarchical structure. CTEs have been effectively used as an optimization fence in many simpler cases, such as this [example](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/43242#note_61416277). @@ -394,8 +394,8 @@ query for every mention of `@alice`. Caching data per transaction can be done using [RequestStore](https://github.com/steveklabnik/request_store) (use `Gitlab::SafeRequestStore` to avoid having to remember to check -`RequestStore.active?`). Caching data in Redis can be done using [Rails' caching -system](https://guides.rubyonrails.org/caching_with_rails.html). +`RequestStore.active?`). Caching data in Redis can be done using +[Rails' caching system](https://guides.rubyonrails.org/caching_with_rails.html). ## Pagination @@ -414,8 +414,7 @@ The main styles of pagination are: The ultimately scalable solution for pagination is to use Keyset-based pagination. However, we don't have support for that at GitLab at that moment. You -can follow the progress looking at [API: Keyset Pagination -](https://gitlab.com/groups/gitlab-org/-/epics/2039). +can follow the progress looking at [API: Keyset Pagination](https://gitlab.com/groups/gitlab-org/-/epics/2039). Take into consideration the following when choosing a pagination strategy: diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index e0e21319f47..64d8b22f1b8 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -45,9 +45,12 @@ work it needs to perform and how long it takes to complete: One exception is a migration that takes longer but is absolutely critical for the application to operate correctly. For example, you might have indices that enforce unique tuples, or that are needed for query performance in critical parts of the application. In cases where the migration would be unacceptably slow, however, a better option might be to guard the feature with a [feature flag](feature_flags/index.md) and perform a post-deployment migration instead. The feature can then be turned on after the migration finishes. + + Migrations used to add new models are also part of these regular schema migrations. The only differences are the Rails command used to generate the migrations and the additional generated files, one for the model and one for the model's spec. 1. [**Post-deployment migrations.**](database/post_deployment_migrations.md) These are Rails migrations in `db/post_migrate` and - run _after_ new application code has been deployed (for GitLab.com after the production deployment has finished). - They can be used for schema changes that aren't critical for the application to operate, or data migrations that take at most a few minutes. + are run independently from the GitLab.com deployments. Pending post migrations are executed on a daily basis at the discretion + of release manager through the [post-deploy migration pipeline](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). + These migrations can be used for schema changes that aren't critical for the application to operate, or data migrations that take at most a few minutes. Common examples for schema changes that should run post-deploy include: - Clean-ups, like removing unused columns. - Adding non-critical indices on high-traffic tables. @@ -88,7 +91,7 @@ Keep in mind that all durations should be measured against GitLab.com. |----|----|---| | Regular migrations | `<= 3 minutes` | A valid exception are changes without which application functionality or performance would be severely degraded and which cannot be delayed. | | Post-deployment migrations | `<= 10 minutes` | A valid exception are schema changes, since they must not happen in background migrations. | -| Background migrations | `> 10 minutes` | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below [`1 second` execution time](query_performance.md#timing-guidelines-for-queries) with cold caches. | +| Background migrations | `> 10 minutes` | Since these are suitable for larger tables, it's not possible to set a precise timing guideline, however, any single query must stay below [`1 second` execution time](database/query_performance.md#timing-guidelines-for-queries) with cold caches. | ## Decide which database to target @@ -108,6 +111,20 @@ bundle exec rails g migration migration_name_here This generates the migration file in `db/migrate`. +### Regular schema migrations to add new models + +To create a new model you can use the following Rails generator: + +```shell +bundle exec rails g model model_name_here +``` + +This will generate: + +- the migration file in `db/migrate` +- the model file in `app/models` +- the spec file in `spec/models` + ## Schema Changes Changes to the schema should be committed to `db/structure.sql`. This @@ -119,7 +136,7 @@ columns manually for existing tables as this causes confusion to other people using `db/structure.sql` generated by Rails. NOTE: -[Creating an index asynchronously requires two merge requests.](adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously) +[Creating an index asynchronously requires two merge requests.](database/adding_database_indexes.md#add-a-migration-to-create-the-index-synchronously) When done, commit the schema change in the merge request that adds the index with `add_concurrent_index`. @@ -245,7 +262,7 @@ When using a single-transaction migration, a transaction holds a database connec for the duration of the migration, so you must make sure the actions in the migration do not take too much time: GitLab.com's production database has a `15s` timeout, so in general, the cumulative execution time in a migration should aim to fit comfortably -in that limit. Singular query timings should fit within the [standard limit](query_performance.md#timing-guidelines-for-queries) +in that limit. Singular query timings should fit within the [standard limit](database/query_performance.md#timing-guidelines-for-queries) In case you need to insert, update, or delete a significant amount of data, you: @@ -268,7 +285,7 @@ which is a "versioned" class. For new migrations, the latest version should be u can be looked up in `Gitlab::Database::Migration::MIGRATION_CLASSES`) to use the latest version of migration helpers. -In this example, we use version 1.0 of the migration class: +In this example, we use version 2.0 of the migration class: ```ruby class TestMigration < Gitlab::Database::Migration[2.0] @@ -580,7 +597,7 @@ end Verify the index is not being used anymore with this Thanos query: ```sql -sum(rate(pg_stat_user_indexes_idx_tup_read{env="gprd", indexrelname="index_ci_name", type="patroni-ci"}[5m])) +sum by (type)(rate(pg_stat_user_indexes_idx_scan{env="gprd", indexrelname="index_groups_on_parent_id_id"}[5m])) ``` Note that it is not necessary to check if the index exists prior to @@ -611,7 +628,7 @@ might not be required, like: Additionally, wide indexes are not required to match all filter criteria of queries, we just need to cover enough columns so that the index lookup has a small enough selectivity. Please review our -[Adding Database indexes](adding_database_indexes.md) guide for more details. +[Adding Database indexes](database/adding_database_indexes.md) guide for more details. When adding an index to a non-empty table make sure to use the method `add_concurrent_index` instead of the regular `add_index` method. @@ -640,7 +657,7 @@ end You must explicitly name indexes that are created with more complex definitions beyond table name, column names, and uniqueness constraint. -Consult the [Adding Database Indexes](adding_database_indexes.md#requirements-for-naming-indexes) +Consult the [Adding Database Indexes](database/adding_database_indexes.md#requirements-for-naming-indexes) guide for more details. If you need to add a unique index, please keep in mind there is the possibility @@ -658,7 +675,7 @@ If a migration requires conditional logic based on the absence or presence of an index, you must test for existence of that index using its name. This helps avoids problems with how Rails compares index definitions, which can lead to unexpected results. For more details, review the -[Adding Database Indexes](adding_database_indexes.md#why-explicit-names-are-required) +[Adding Database Indexes](database/adding_database_indexes.md#why-explicit-names-are-required) guide. The easiest way to test for existence of an index by name is to use the @@ -739,6 +756,21 @@ If a backport adding a column with a default value is needed for %12.9 or earlie it should use `add_column_with_default` helper. If a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) is involved, backporting to %12.9 is contraindicated. +## Removing the column default for non-nullable columns + +If you have added a non-nullable column, and used the default value to populate +existing data, you need to keep that default value around until at least after +the application code is updated. You cannot remove the default value in the +same migration, as the migrations run before the model code is updated and +models will have an old schema cache, meaning they won't know about this column +and won't be able to set it. In this case it's recommended to: + +1. Add the column with default value in a normal migration. +1. Remove the default in a post-deployment migration. + +The post-deployment migration happens after the application restarts, +ensuring the new column has been discovered. + ## Changing the column default One might think that changing a default column with `change_column_default` is an @@ -1196,8 +1228,8 @@ If using a model in the migrations, you should first [clear the column cache](https://api.rubyonrails.org/classes/ActiveRecord/ModelSchema/ClassMethods.html#method-i-reset_column_information) using `reset_column_information`. -If using a model that leverages single table inheritance (STI), there are [special -considerations](single_table_inheritance.md#in-migrations). +If using a model that leverages single table inheritance (STI), there are +[special considerations](database/single_table_inheritance.md#in-migrations). This avoids problems where a column that you are using was altered and cached in a previous migration. diff --git a/doc/development/module_with_instance_variables.md b/doc/development/module_with_instance_variables.md index 0f910f20534..8e39186d396 100644 --- a/doc/development/module_with_instance_variables.md +++ b/doc/development/module_with_instance_variables.md @@ -35,12 +35,10 @@ one of the variables. Everything could touch anything. People are saying multiple inheritance is bad. Mixing multiple modules with multiple instance variables scattering everywhere suffer from the same issue. The same applies to `ActiveSupport::Concern`. See: -[Consider replacing concerns with dedicated classes & composition]( -https://gitlab.com/gitlab-org/gitlab/-/issues/16270) +[Consider replacing concerns with dedicated classes & composition](https://gitlab.com/gitlab-org/gitlab/-/issues/16270) There's also a similar idea: -[Use decorators and interface segregation to solve overgrowing models problem]( -https://gitlab.com/gitlab-org/gitlab/-/issues/14235) +[Use decorators and interface segregation to solve overgrowing models problem](https://gitlab.com/gitlab-org/gitlab/-/issues/14235) Note that `included` doesn't solve the whole issue. They define the dependencies, but they still allow each modules to talk implicitly via the diff --git a/doc/development/multi_version_compatibility.md b/doc/development/multi_version_compatibility.md index bdab92f5185..0f3531cf5dd 100644 --- a/doc/development/multi_version_compatibility.md +++ b/doc/development/multi_version_compatibility.md @@ -270,7 +270,7 @@ and set this column to `false`. The old servers were still updating the old colu that updated the new column from the old one. For the new servers though, they were only updating the new column and that same trigger was now working against us and setting it back to the wrong value. -For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9176). +For more information, see [the relevant issue](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/9176). ### Sidebar wasn't loading for some users diff --git a/doc/development/namespaces_storage_statistics.md b/doc/development/namespaces_storage_statistics.md index e5263288210..75e79d1f693 100644 --- a/doc/development/namespaces_storage_statistics.md +++ b/doc/development/namespaces_storage_statistics.md @@ -1,193 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/namespaces_storage_statistics.md' +remove_date: '2022-11-05' --- -# Database case study: Namespaces storage statistics +This document was moved to [another location](database/namespaces_storage_statistics.md). -## Introduction - -On [Storage and limits management for groups](https://gitlab.com/groups/gitlab-org/-/epics/886), -we want to facilitate a method for easily viewing the amount of -storage consumed by a group, and allow easy management. - -## Proposal - -1. Create a new ActiveRecord model to hold the namespaces' statistics in an aggregated form (only for root [namespaces](../user/group/index.md#namespaces)). -1. Refresh the statistics in this model every time a project belonging to this namespace is changed. - -## Problem - -In GitLab, we update the project storage statistics through a -[callback](https://gitlab.com/gitlab-org/gitlab/-/blob/4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e/app/models/project.rb#L97) -every time the project is saved. - -The summary of those statistics per namespace is then retrieved -by [`Namespaces#with_statistics`](https://gitlab.com/gitlab-org/gitlab/-/blob/4ab54c2233e91f60a80e5b6fa2181e6899fdcc3e/app/models/namespace.rb#L70) scope. Analyzing this query we noticed that: - -- It takes up to `1.2` seconds for namespaces with over `15k` projects. -- It can't be analyzed with [ChatOps](chatops_on_gitlabcom.md), as it times out. - -Additionally, the pattern that is currently used to update the project statistics -(the callback) doesn't scale adequately. It is currently one of the largest -[database queries transactions on production](https://gitlab.com/gitlab-org/gitlab/-/issues/29070) -that takes the most time overall. We can't add one more query to it as -it increases the transaction's length. - -Because of all of the above, we can't apply the same pattern to store -and update the namespaces statistics, as the `namespaces` table is one -of the largest tables on GitLab.com. Therefore we needed to find a performant and -alternative method. - -## Attempts - -### Attempt A: PostgreSQL materialized view - -Model can be updated through a refresh strategy based on a project routes SQL and a [materialized view](https://www.postgresql.org/docs/11/rules-materializedviews.html): - -```sql -SELECT split_part("rs".path, '/', 1) as root_path, - COALESCE(SUM(ps.storage_size), 0) AS storage_size, - COALESCE(SUM(ps.repository_size), 0) AS repository_size, - COALESCE(SUM(ps.wiki_size), 0) AS wiki_size, - COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size, - COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size, - COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size, - COALESCE(SUM(ps.packages_size), 0) AS packages_size, - COALESCE(SUM(ps.snippets_size), 0) AS snippets_size, - COALESCE(SUM(ps.uploads_size), 0) AS uploads_size -FROM "projects" - INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' - INNER JOIN project_statistics ps ON ps.project_id = projects.id -GROUP BY root_path -``` - -We could then execute the query with: - -```sql -REFRESH MATERIALIZED VIEW root_namespace_storage_statistics; -``` - -While this implied a single query update (and probably a fast one), it has some downsides: - -- Materialized views syntax varies from PostgreSQL and MySQL. While this feature was worked on, MySQL was still supported by GitLab. -- Rails does not have native support for materialized views. We'd need to use a specialized gem to take care of the management of the database views, which implies additional work. - -### Attempt B: An update through a CTE - -Similar to Attempt A: Model update done through a refresh strategy with a [Common Table Expression](https://www.postgresql.org/docs/9.1/queries-with.html) - -```sql -WITH refresh AS ( - SELECT split_part("rs".path, '/', 1) as root_path, - COALESCE(SUM(ps.storage_size), 0) AS storage_size, - COALESCE(SUM(ps.repository_size), 0) AS repository_size, - COALESCE(SUM(ps.wiki_size), 0) AS wiki_size, - COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size, - COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size, - COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size, - COALESCE(SUM(ps.packages_size), 0) AS packages_size, - COALESCE(SUM(ps.snippets_size), 0) AS snippets_size, - COALESCE(SUM(ps.uploads_size), 0) AS uploads_size - FROM "projects" - INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' - INNER JOIN project_statistics ps ON ps.project_id = projects.id - GROUP BY root_path) -UPDATE namespace_storage_statistics -SET storage_size = refresh.storage_size, - repository_size = refresh.repository_size, - wiki_size = refresh.wiki_size, - lfs_objects_size = refresh.lfs_objects_size, - build_artifacts_size = refresh.build_artifacts_size, - pipeline_artifacts_size = refresh.pipeline_artifacts_size, - packages_size = refresh.packages_size, - snippets_size = refresh.snippets_size, - uploads_size = refresh.uploads_size -FROM refresh - INNER JOIN routes rs ON rs.path = refresh.root_path AND rs.source_type = 'Namespace' -WHERE namespace_storage_statistics.namespace_id = rs.source_id -``` - -Same benefits and downsides as attempt A. - -### Attempt C: Get rid of the model and store the statistics on Redis - -We could get rid of the model that stores the statistics in aggregated form and instead use a Redis Set. -This would be the [boring solution](https://about.gitlab.com/handbook/values/#boring-solutions) and the fastest one -to implement, as GitLab already includes Redis as part of its [Architecture](architecture.md#redis). - -The downside of this approach is that Redis does not provide the same persistence/consistency guarantees as PostgreSQL, -and this is information we can't afford to lose in a Redis failure. - -### Attempt D: Tag the root namespace and its child namespaces - -Directly relate the root namespace to its child namespaces, so -whenever a namespace is created without a parent, this one is tagged -with the root namespace ID: - -| ID | root ID | parent ID | -|:---|:--------|:----------| -| 1 | 1 | NULL | -| 2 | 1 | 1 | -| 3 | 1 | 2 | - -To aggregate the statistics inside a namespace we'd execute something like: - -```sql -SELECT COUNT(...) -FROM projects -WHERE namespace_id IN ( - SELECT id - FROM namespaces - WHERE root_id = X -) -``` - -Even though this approach would make aggregating much easier, it has some major downsides: - -- We'd have to migrate **all namespaces** by adding and filling a new column. Because of the size of the table, dealing with time/cost would be significant. The background migration would take approximately `153h`, see <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29772>. -- Background migration has to be shipped one release before, delaying the functionality by another milestone. - -### Attempt E (final): Update the namespace storage statistics asynchronously - -This approach consists of continuing to use the incremental statistics updates we already have, -but we refresh them through Sidekiq jobs and in different transactions: - -1. Create a second table (`namespace_aggregation_schedules`) with two columns `id` and `namespace_id`. -1. Whenever the statistics of a project changes, insert a row into `namespace_aggregation_schedules` - - We don't insert a new row if there's already one related to the root namespace. - - Keeping in mind the length of the transaction that involves updating `project_statistics`(<https://gitlab.com/gitlab-org/gitlab/-/issues/29070>), the insertion should be done in a different transaction and through a Sidekiq Job. -1. After inserting the row, we schedule another worker to be executed asynchronously at two different moments: - - One enqueued for immediate execution and another one scheduled in `1.5h` hours. - - We only schedule the jobs, if we can obtain a `1.5h` lease on Redis on a key based on the root namespace ID. - - If we can't obtain the lease, it indicates there's another aggregation already in progress, or scheduled in no more than `1.5h`. -1. This worker will: - - Update the root namespace storage statistics by querying all the namespaces through a service. - - Delete the related `namespace_aggregation_schedules` after the update. -1. Another Sidekiq job is also included to traverse any remaining rows on the `namespace_aggregation_schedules` table and schedule jobs for every pending row. - - This job is scheduled with cron to run every night (UTC). - -This implementation has the following benefits: - -- All the updates are done asynchronously, so we're not increasing the length of the transactions for `project_statistics`. -- We're doing the update in a single SQL query. -- It is compatible with PostgreSQL and MySQL. -- No background migration required. - -The only downside of this approach is that namespaces' statistics are updated up to `1.5` hours after the change is done, -which means there's a time window in which the statistics are inaccurate. Because we're still not -[enforcing storage limits](https://gitlab.com/gitlab-org/gitlab/-/issues/17664), this is not a major problem. - -## Conclusion - -Updating the storage statistics asynchronously, was the less problematic and -performant approach of aggregating the root namespaces. - -All the details regarding this use case can be found on: - -- <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62214> -- Merge Request with the implementation: <https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/28996> - -Performance of the namespace storage statistics were measured in staging and production (GitLab.com). All results were posted -on <https://gitlab.com/gitlab-org/gitlab-foss/-/issues/64092>: No problem has been reported so far. +<!-- This redirect file can be deleted after <2022-11-05>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index 65485104efe..9575acd20c7 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -1,52 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/accessibility.md' +remove_date: '2022-11-15' --- -# Accessibility +This document was moved to [another location](../../fe_guide/accessibility.md). -Using semantic HTML plays a key role when it comes to accessibility. - -## Accessible Rich Internet Applications - ARIA - -WAI-ARIA (the Accessible Rich Internet Applications specification) defines a way to make Web content and Web applications more accessible to people with disabilities. - -The W3C recommends [using semantic elements](https://www.w3.org/TR/using-aria/#notes2) as the primary method to achieve accessibility rather than adding aria attributes. Adding aria attributes should be seen as a secondary method for creating accessible elements. - -### Role - -The `role` attribute describes the role the element plays in the context of the document. - -Review the list of [WAI-ARIA roles](https://www.w3.org/TR/wai-aria-1.1/#landmark_roles). - -## Icons - -When using icons or images that aren't absolutely needed to understand the context, we should use `aria-hidden="true"`. - -On the other hand, if an icon is crucial to understand the context we should do one of the following: - -1. Use `aria-label` in the element with a meaningful description -1. Use `aria-labelledby` to point to an element that contains the explanation for that icon - -## Form inputs - -In forms we should use the `for` attribute in the label statement: - -```html -<div> - <label for="name">Fill in your name:</label> - <input type="text" id="name" name="name"> -</div> -``` - -## Testing - -1. On MacOS you can use [VoiceOver](https://www.apple.com/accessibility/vision/) by pressing `cmd+F5`. -1. On Windows you can use [Narrator](https://www.microsoft.com/en-us/accessibility/windows) by pressing Windows logo key + Control + Enter. - -## Online resources - -- [Chrome Accessibility Developer Tools](https://github.com/GoogleChrome/accessibility-developer-tools) for testing accessibility -- [Audit Rules Page](https://github.com/GoogleChrome/accessibility-developer-tools/wiki/Audit-Rules) for best practices -- [Lighthouse Accessibility Score](https://web.dev/performance-scoring/) for accessibility audits +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/components.md b/doc/development/new_fe_guide/development/components.md index ec714c9c26f..9ad742272d1 100644 --- a/doc/development/new_fe_guide/development/components.md +++ b/doc/development/new_fe_guide/development/components.md @@ -1,27 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Components +This document was moved to [another location](../../fe_guide/index.md). -## Graphs - -We have a lot of graphing libraries in our codebase to render graphs. In an effort to improve maintainability, new graphs should use [D3.js](https://d3js.org/). If a new graph is fairly simple, consider implementing it in SVGs or HTML5 canvas. - -We chose D3 as our library going forward because of the following features: - -- [Tree shaking webpack capabilities](https://github.com/d3/d3/blob/master/CHANGES.md#changes-in-d3-40). -- [Compatible with vue.js as well as vanilla JavaScript](https://github.com/d3/d3/blob/master/CHANGES.md#changes-in-d3-40). - -D3 is very popular across many projects outside of GitLab: - -- [The New York Times](https://archive.nytimes.com/www.nytimes.com/interactive/2012/02/13/us/politics/2013-budget-proposal-graphic.html) -- [plot.ly](https://plotly.com/) -- [Ayoa](https://www.ayoa.com/previously-droptask/) - -Within GitLab, D3 has been used for the following notable features - -- [Prometheus graphs](../../../user/project/integrations/prometheus.md) -- Contribution calendars +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/index.md b/doc/development/new_fe_guide/development/index.md index 5922c3aeeed..9ad742272d1 100644 --- a/doc/development/new_fe_guide/development/index.md +++ b/doc/development/new_fe_guide/development/index.md @@ -1,23 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Development +This document was moved to [another location](../../fe_guide/index.md). -## [Components](components.md) - -Documentation on existing components and how to best create a new component. - -## [Accessibility](accessibility.md) - -Learn how to implement an accessible frontend. - -## [Performance](performance.md) - -Learn how to keep our frontend performant. - -## [Testing](../../testing_guide/frontend_testing.md) - -Learn how to keep our frontend tested. +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/development/performance.md b/doc/development/new_fe_guide/development/performance.md index ee853942cb9..c72f3ded896 100644 --- a/doc/development/new_fe_guide/development/performance.md +++ b/doc/development/new_fe_guide/development/performance.md @@ -1,22 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/performance.md' +remove_date: '2022-11-15' --- -# Performance +This document was moved to [another location](../../fe_guide/performance.md). -## Monitoring - -We have a performance dashboard available in one of our [Grafana instances](https://dashboards.gitlab.net/d/000000043/sitespeed-page-summary?orgId=1). This dashboard automatically aggregates metric data from [sitespeed.io](https://www.sitespeed.io/) every 4 hours. These changes are displayed after a set number of pages are aggregated. - -These pages can be found inside text files in the [`sitespeed-measurement-setup` repository](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup) called [`gitlab`](https://gitlab.com/gitlab-org/frontend/sitespeed-measurement-setup/-/tree/master/gitlab) -Any frontend engineer can contribute to this dashboard. They can contribute by adding or removing URLs of pages to the text files. The changes are pushed live on the next scheduled run after the changes are merged into `main`. - -There are 3 recommended high impact metrics (core web vitals) to review on each page: - -- [Largest Contentful Paint](https://web.dev/lcp/) -- [First Input Delay](https://web.dev/fid/) -- [Cumulative Layout Shift](https://web.dev/cls/) - -For these metrics, lower numbers are better as it means that the website is more performant. +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/index.md b/doc/development/new_fe_guide/index.md index 4d4098844b2..83c1db696b4 100644 --- a/doc/development/new_fe_guide/index.md +++ b/doc/development/new_fe_guide/index.md @@ -1,22 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Frontend Development Guidelines +This document was moved to [another location](../fe_guide/index.md). -This guide contains all the information to successfully contribute to the GitLab frontend. -This is a living document, and we welcome contributions, feedback, and suggestions. - -## [Development](development/index.md) - -Guidance on topics related to development. - -## [Modules](modules/index.md) - -Learn about all the internal JavaScript modules that make up our frontend. - -## [Tips](tips.md) - -Tips from our frontend team to develop more efficiently and effectively. +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/dirty_submit.md b/doc/development/new_fe_guide/modules/dirty_submit.md index 6e1062aa72e..9ad742272d1 100644 --- a/doc/development/new_fe_guide/modules/dirty_submit.md +++ b/doc/development/new_fe_guide/modules/dirty_submit.md @@ -1,28 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Dirty Submit +This document was moved to [another location](../../fe_guide/index.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/21115) in GitLab 11.3. - -## Summary - -Prevent submitting forms with no changes. - -Currently handles `input`, `textarea` and `select` elements. - -Also, see [the code](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/dirty_submit/) -within the GitLab project. - -## Usage - -```javascript -import dirtySubmitFactory from './dirty_submit/dirty_submit_form'; - -new DirtySubmitForm(document.querySelector('form')); -// or -new DirtySubmitForm(document.querySelectorAll('form')); -``` +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/index.md b/doc/development/new_fe_guide/modules/index.md index a9bdcda4a2d..9ad742272d1 100644 --- a/doc/development/new_fe_guide/modules/index.md +++ b/doc/development/new_fe_guide/modules/index.md @@ -1,15 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Modules +This document was moved to [another location](../../fe_guide/index.md). -- [DirtySubmit](dirty_submit.md) - - Disable form submits until there are unsaved changes. - -- [Merge Request widget extensions](widget_extensions.md) - - Easily add extensions into the merge request widget +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/modules/widget_extensions.md b/doc/development/new_fe_guide/modules/widget_extensions.md index 4bae0ac70c4..3741ee8c38a 100644 --- a/doc/development/new_fe_guide/modules/widget_extensions.md +++ b/doc/development/new_fe_guide/modules/widget_extensions.md @@ -1,355 +1,11 @@ --- -stage: Create -group: Code Review -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../fe_guide/merge_request_widget_extensions.md' +remove_date: '2022-11-15' --- -# Merge request widget extensions **(FREE)** +This document was moved to [another location](../../fe_guide/merge_request_widget_extensions.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44616) in GitLab 13.6. - -## Summary - -Extensions in the merge request widget enable you to add new features -into the merge request widget that match the design framework. -With extensions we get a lot of benefits out of the box without much effort required, like: - -- A consistent look and feel. -- Tracking when the extension is opened. -- Virtual scrolling for performance. - -## Usage - -To use extensions you must first create a new extension object to fetch the -data to render in the extension. For a working example, refer to the example file in -`app/assets/javascripts/vue_merge_request_widget/extensions/issues.js`. - -The basic object structure: - -```javascript -export default { - name: '', // Required: This helps identify the widget - props: [], // Required: Props passed from the widget state - i18n: { // Required: Object to hold i18n text - label: '', // Required: Used for tooltips and aria-labels - loading: '', // Required: Loading text for when data is loading - }, - expandEvent: '', // Optional: RedisHLL event name to track expanding content - enablePolling: false, // Optional: Tells extension to poll for data - modalComponent: null, // Optional: The component to use for the modal - computed: { - summary(data) {}, // Required: Level 1 summary text - statusIcon(data) {}, // Required: Level 1 status icon - tertiaryButtons() {}, // Optional: Level 1 action buttons - shouldCollapse() {}, // Optional: Add logic to determine if the widget can expand or not - }, - methods: { - fetchCollapsedData(props) {}, // Required: Fetches data required for collapsed state - fetchFullData(props) {}, // Required: Fetches data for the full expanded content - fetchMultiData() {}, // Optional: Works in conjunction with `enablePolling` and allows polling multiple endpoints - }, -}; -``` - -By following the same data structure, each extension can follow the same registering structure, -but each extension can manage its data sources. - -After creating this structure, you must register it. You can register the extension at any -point _after_ the widget has been created. To register a extension: - -```javascript -// Import the register method -import { registerExtension } from '~/vue_merge_request_widget/components/extensions'; - -// Import the new extension -import issueExtension from '~/vue_merge_request_widget/extensions/issues'; - -// Register the imported extension -registerExtension(issueExtension); -``` - -## Data fetching - -Each extension must fetch data. Fetching is handled when registering the extension, -not by the core component itself. This approach allows for various different -data fetching methods to be used, such as GraphQL or REST API calls. - -### API calls - -For performance reasons, it is best if the collapsed state fetches only the data required to -render the collapsed state. This fetching happens within the `fetchCollapsedData` method. -This method is called with the props as an argument, so you can easily access -any paths set in the state. - -To allow the extension to set the data, this method **must** return the data. No -special formatting is required. When the extension receives this data, -it is set to `collapsedData`. You can access `collapsedData` in any computed property or -method. - -When the user clicks **Expand**, the `fetchFullData` method is called. This method -also gets called with the props as an argument. This method **must** also return -the full data. However, this data needs to be correctly formatted to match the format -mentioned in the data structure section. - -#### Technical debt - -For some of the current extensions, there is no split in data fetching. All the data -is fetched through the `fetchCollapsedData` method. While less performant, -it allows for faster iteration. - -To handle this the `fetchFullData` returns the data set through -the `fetchCollapsedData` method call. In these cases, the `fetchFullData` must -return a promise: - -```javascript -fetchCollapsedData() { - return ['Some data']; -}, -fetchFullData() { - return Promise.resolve(this.collapsedData) -}, -``` - -### Data structure - -The data returned from `fetchFullData` must match the format below. This format -allows the core component to render the data in a way that matches -the design framework. Any text properties can use the styling placeholders -mentioned below: - -```javascript -{ - id: data.id, // Required: ID used as a key for each row - header: 'Header' || ['Header', 'sub-header'], // Required: String or array can be used for the header text - text: '', // Required: Main text for the row - subtext: '', // Optional: Smaller sub-text to be displayed below the main text - icon: { // Optional: Icon object - name: EXTENSION_ICONS.success, // Required: The icon name for the row - }, - badge: { // Optional: Badge displayed after text - text: '', // Required: Text to be displayed inside badge - variant: '', // Optional: GitLab UI badge variant, defaults to info - }, - link: { // Optional: Link to a URL displayed after text - text: '', // Required: Text of the link - href: '', // Optional: URL for the link - }, - modal: { // Optional: Link to open a modal displayed after text - text: '', // Required: Text of the link - onClick: () => {} // Optional: Function to run when link is clicked, i.e. to set this.modalData - } - actions: [], // Optional: Action button for row - children: [], // Optional: Child content to render, structure matches the same structure -} -``` - -### Polling - -To enable polling for an extension, an options flag must be present in the extension: - -```javascript -export default { - //... - enablePolling: true -}; -``` - -This flag tells the base component we should poll the `fetchCollapsedData()` -defined in the extension. Polling stops if the response has data, or if an error is present. - -When writing the logic for `fetchCollapsedData()`, a complete Axios response must be returned -from the method. The polling utility needs data like polling headers to work correctly: - -```javascript -export default { - //... - enablePolling: true - methods: { - fetchCollapsedData() { - return axios.get(this.reportPath) - }, - }, -}; -``` - -Most of the time the data returned from the extension's endpoint is not in the format -the UI needs. We must format the data before setting the collapsed data in the base component. - -If the computed property `summary` can rely on `collapsedData`, you can format the data -when `fetchFullData` is invoked: - -```javascript -export default { - //... - enablePolling: true - methods: { - fetchCollapsedData() { - return axios.get(this.reportPath) - }, - fetchFullData() { - return Promise.resolve(this.prepareReports()); - }, - // custom method - prepareReports() { - // unpack values from collapsedData - const { new_errors, existing_errors, resolved_errors } = this.collapsedData; - - // perform data formatting - - return [...newErrors, ...existingErrors, ...resolvedErrors] - } - }, -}; -``` - -If the extension relies on `collapsedData` being formatted before invoking `fetchFullData()`, -then `fetchCollapsedData()` must return the Axios response as well as the formatted data: - -```javascript -export default { - //... - enablePolling: true - methods: { - fetchCollapsedData() { - return axios.get(this.reportPath).then(res => { - const formattedData = this.prepareReports(res.data) - - return { - ...res, - data: formattedData, - } - }) - }, - // Custom method - prepareReports() { - // Unpack values from collapsedData - const { new_errors, existing_errors, resolved_errors } = this.collapsedData; - - // Perform data formatting - - return [...newErrors, ...existingErrors, ...resolvedErrors] - } - }, -}; -``` - -If the extension needs to poll multiple endpoints at the same time, then `fetchMultiData` -can be used to return an array of functions. A new `poll` object is created for each -endpoint and they are polled separately. After all endpoints are resolved, polling is -stopped and `setCollapsedData` is called with an array of `response.data`. - -```javascript -export default { - //... - enablePolling: true - methods: { - fetchMultiData() { - return [ - () => axios.get(this.reportPath1), - () => axios.get(this.reportPath2), - () => axios.get(this.reportPath3) - }, - }, -}; -``` - -**Important** The function needs to return a `Promise` that resolves the `response` object. -The implementation relies on the `POLL-INTERVAL` header to keep polling, therefore it is -important not to alter the status code and headers. - -### Errors - -If `fetchCollapsedData()` or `fetchFullData()` methods throw an error: - -- The loading state of the extension is updated to `LOADING_STATES.collapsedError` - and `LOADING_STATES.expandedError` respectively. -- The extensions header displays an error icon and updates the text to be either: - - The text defined in `$options.i18n.error`. - - "Failed to load" if `$options.i18n.error` is not defined. -- The error is sent to Sentry to log that it occurred. - -To customise the error text, add it to the `i18n` object in your extension: - -```javascript -export default { - //... - i18n: { - //... - error: __('Your error text'), - }, -}; -``` - -## Icons - -Level 1 and all subsequent levels can have their own status icons. To keep with -the design framework, import the `EXTENSION_ICONS` constant -from the `constants.js` file: - -```javascript -import { EXTENSION_ICONS } from '~/vue_merge_request_widget/constants.js'; -``` - -This constant has the below icons available for use. Per the design framework, -only some of these icons should be used on level 1: - -- `failed` -- `warning` -- `success` -- `neutral` -- `error` -- `notice` -- `severityCritical` -- `severityHigh` -- `severityMedium` -- `severityLow` -- `severityInfo` -- `severityUnknown` - -## Text styling - -Any area that has text can be styled with the placeholders below. This -technique follows the same technique as `sprintf`. However, instead of specifying -these through `sprintf`, the extension does this automatically. - -Every placeholder contains starting and ending tags. For example, `success` uses -`Hello %{success_start}world%{success_end}`. The extension then -adds the start and end tags with the correct styling classes. - -| Placeholder | Style | -|---|---| -| success | `gl-font-weight-bold gl-text-green-500` | -| danger | `gl-font-weight-bold gl-text-red-500` | -| critical | `gl-font-weight-bold gl-text-red-800` | -| same | `gl-font-weight-bold gl-text-gray-700` | -| strong | `gl-font-weight-bold` | -| small | `gl-font-sm` | - -## Action buttons - -You can add action buttons to all level 1 and 2 in each extension. These buttons -are meant as a way to provide links or actions for each row: - -- Action buttons for level 1 can be set through the `tertiaryButtons` computed property. - This property should return an array of objects for each action button. -- Action buttons for level 2 can be set by adding the `actions` key to the level 2 rows object. - The value for this key must also be an array of objects for each action button. - -Links must follow this structure: - -```javascript -{ - text: 'Click me', - href: this.someLinkHref, - target: '_blank', // Optional -} -``` - -For internal action buttons, follow this structure: - -```javascript -{ - text: 'Click me', - onClick() {} -} -``` +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/new_fe_guide/tips.md b/doc/development/new_fe_guide/tips.md index 5d4c0fc019f..83c1db696b4 100644 --- a/doc/development/new_fe_guide/tips.md +++ b/doc/development/new_fe_guide/tips.md @@ -1,35 +1,11 @@ --- -stage: none -group: Development -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../fe_guide/index.md' +remove_date: '2022-11-15' --- -# Tips +This document was moved to [another location](../fe_guide/index.md). -## Clearing production compiled assets - -To clear production compiled assets created with `yarn webpack-prod` you can run: - -```shell -yarn clean -``` - -## Creating feature flags in development - -The process for creating a feature flag is the same as [enabling a feature flag in development](../feature_flags/index.md#enabling-a-feature-flag-locally-in-development). - -Your feature flag can now be: - -- [Made available to the frontend](../feature_flags/index.md#frontend) via the `gon` -- Queried in [tests](../feature_flags/index.md#feature-flags-in-tests) -- Queried in HAML templates and Ruby files via the `Feature.enabled?(:my_shiny_new_feature_flag)` method - -### More on feature flags - -- [Deleting a feature flag](../../api/features.md#delete-a-feature) -- [Manage feature flags](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/) -- [Feature flags API](../../api/features.md) - -## Running tests locally - -This can be done as outlined by the [frontend testing guide](../testing_guide/frontend_testing.md#running-frontend-tests). +<!-- This redirect file can be deleted after <2022-11-15>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/ordering_table_columns.md b/doc/development/ordering_table_columns.md index 7cd3d4fb208..b665cb0d4c7 100644 --- a/doc/development/ordering_table_columns.md +++ b/doc/development/ordering_table_columns.md @@ -1,152 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/ordering_table_columns.md' +remove_date: '2022-11-04' --- -# Ordering Table Columns in PostgreSQL +This document was moved to [another location](database/ordering_table_columns.md). -For GitLab we require that columns of new tables are ordered to use the -least amount of space. An easy way of doing this is to order them based on the -type size in descending order with variable sizes (`text`, `varchar`, arrays, -`json`, `jsonb`, and so on) at the end. - -Similar to C structures the space of a table is influenced by the order of -columns. This is because the size of columns is aligned depending on the type of -the following column. Let's consider an example: - -- `id` (integer, 4 bytes) -- `name` (text, variable) -- `user_id` (integer, 4 bytes) - -The first column is a 4-byte integer. The next is text of variable length. The -`text` data type requires 1-word alignment, and on 64-bit platform, 1 word is 8 -bytes. To meet the alignment requirements, four zeros are to be added right -after the first column, so `id` occupies 4 bytes, then 4 bytes of alignment -padding, and only next `name` is being stored. Therefore, in this case, 8 bytes -are spent for storing a 4-byte integer. - -The space between rows is also subject to alignment padding. The `user_id` -column takes only 4 bytes, and on 64-bit platform, 4 zeroes are added for -alignment padding, to allow storing the next row beginning with the "clear" word. - -As a result, the actual size of each column would be (omitting variable length -data and 24-byte tuple header): 8 bytes, variable, 8 bytes. This means that -each row requires at least 16 bytes for the two 4-byte integers. If a table -has a few rows this is not an issue. However, once you start storing millions of -rows you can save space by using a different order. For the above example, the -ideal column order would be the following: - -- `id` (integer, 4 bytes) -- `user_id` (integer, 4 bytes) -- `name` (text, variable) - -or - -- `name` (text, variable) -- `id` (integer, 4 bytes) -- `user_id` (integer, 4 bytes) - -In these examples, the `id` and `user_id` columns are packed together, which -means we only need 8 bytes to store _both_ of them. This in turn means each row -requires 8 bytes less space. - -Since Ruby on Rails 5.1, the default data type for IDs is `bigint`, which uses 8 bytes. -We are using `integer` in the examples to showcase a more realistic reordering scenario. - -## Type Sizes - -While the [PostgreSQL documentation](https://www.postgresql.org/docs/current/datatype.html) contains plenty -of information we list the sizes of common types here so it's easier to -look them up. Here "word" refers to the word size, which is 4 bytes for a 32 -bits platform and 8 bytes for a 64 bits platform. - -| Type | Size | Alignment needed | -|:-----------------|:-------------------------------------|:-----------| -| `smallint` | 2 bytes | 1 word | -| `integer` | 4 bytes | 1 word | -| `bigint` | 8 bytes | 8 bytes | -| `real` | 4 bytes | 1 word | -| `double precision` | 8 bytes | 8 bytes | -| `boolean` | 1 byte | not needed | -| `text` / `string` | variable, 1 byte plus the data | 1 word | -| `bytea` | variable, 1 or 4 bytes plus the data | 1 word | -| `timestamp` | 8 bytes | 8 bytes | -| `timestamptz` | 8 bytes | 8 bytes | -| `date` | 4 bytes | 1 word | - -A "variable" size means the actual size depends on the value being stored. If -PostgreSQL determines this can be embedded directly into a row it may do so, but -for very large values it stores the data externally and store a pointer (of -1 word in size) in the column. Because of this variable sized columns should -always be at the end of a table. - -## Real Example - -Let's use the `events` table as an example, which currently has the following -layout: - -| Column | Type | Size | -|:--------------|:----------------------------|:---------| -| `id` | integer | 4 bytes | -| `target_type` | character varying | variable | -| `target_id` | integer | 4 bytes | -| `title` | character varying | variable | -| `data` | text | variable | -| `project_id` | integer | 4 bytes | -| `created_at` | timestamp without time zone | 8 bytes | -| `updated_at` | timestamp without time zone | 8 bytes | -| `action` | integer | 4 bytes | -| `author_id` | integer | 4 bytes | - -After adding padding to align the columns this would translate to columns being -divided into fixed size chunks as follows: - -| Chunk Size | Columns | -|:-----------|:----------------------| -| 8 bytes | `id` | -| variable | `target_type` | -| 8 bytes | `target_id` | -| variable | `title` | -| variable | `data` | -| 8 bytes | `project_id` | -| 8 bytes | `created_at` | -| 8 bytes | `updated_at` | -| 8 bytes | `action`, `author_id` | - -This means that excluding the variable sized data and tuple header, we need at -least 8 * 6 = 48 bytes per row. - -We can optimise this by using the following column order instead: - -| Column | Type | Size | -|:--------------|:----------------------------|:---------| -| `created_at` | timestamp without time zone | 8 bytes | -| `updated_at` | timestamp without time zone | 8 bytes | -| `id` | integer | 4 bytes | -| `target_id` | integer | 4 bytes | -| `project_id` | integer | 4 bytes | -| `action` | integer | 4 bytes | -| `author_id` | integer | 4 bytes | -| `target_type` | character varying | variable | -| `title` | character varying | variable | -| `data` | text | variable | - -This would produce the following chunks: - -| Chunk Size | Columns | -|:-----------|:-----------------------| -| 8 bytes | `created_at` | -| 8 bytes | `updated_at` | -| 8 bytes | `id`, `target_id` | -| 8 bytes | `project_id`, `action` | -| 8 bytes | `author_id` | -| variable | `target_type` | -| variable | `title` | -| variable | `data` | - -Here we only need 40 bytes per row excluding the variable sized data and 24-byte -tuple header. 8 bytes being saved may not sound like much, but for tables as -large as the `events` table it does begin to matter. For example, when storing -80 000 000 rows this translates to a space saving of at least 610 MB, all by -just changing the order of a few columns. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/pages/index.md b/doc/development/pages/index.md index 02019db48ba..af9d4d33683 100644 --- a/doc/development/pages/index.md +++ b/doc/development/pages/index.md @@ -10,8 +10,8 @@ description: "GitLab's development guidelines for GitLab Pages" ## Configuring GitLab Pages hostname -GitLab Pages needs a hostname or domain, as each different GitLab Pages site is accessed via a -subdomain. GitLab Pages hostname can be set in different manners: +GitLab Pages need a hostname or domain, as each different GitLab Pages site is accessed via a +subdomain. You can set the GitLab Pages hostname: - [Without wildcard, editing your hosts file](#without-wildcard-editing-your-hosts-file). - [With DNS wildcard alternatives](#with-dns-wildcard-alternatives). @@ -96,7 +96,7 @@ it with commands like: ### Running GitLab Pages manually -You can also build and start the app independent of GDK processes management. +You can also build and start the app independently of GDK processes management. For any changes in the code, you must run `make` to build the app. It's best to just always run it before you start the app. It's quick to build so don't worry! @@ -114,9 +114,9 @@ FIPS_MODE=1 make && ./gitlab-pages -config=gitlab-pages.conf ### Creating GitLab Pages site To build a GitLab Pages site locally you must -[configure `gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/runner.md) +[configure `gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/runner.md). -Check the [user manual](../../user/project/pages/index.md). +For more information, refer to the [user manual](../../user/project/pages/index.md). ### Enabling access control @@ -125,8 +125,9 @@ who have access to your GitLab project. GitLab Pages access control is disabled by default. To enable it: -1. Enable the GitLab Pages access control in GitLab itself, which can be done by either: - - If you're not using GDK, editing `gitlab.yml`: +1. Enable the GitLab Pages access control in GitLab itself. You can do this in two ways: + + - If you're not using GDK, edit `gitlab.yml`: ```yaml # gitlab/config/gitlab.yml @@ -134,7 +135,7 @@ GitLab Pages access control is disabled by default. To enable it: access_control: true ``` - - Editing `gdk.yml` if you're using GDK: + - If you're using GDK, edit `gdk.yml`: ```yaml # $GDK_ROOT/gdk.yml @@ -149,8 +150,9 @@ GitLab Pages access control is disabled by default. To enable it: 1. Create an [Instance-wide OAuth application](../../integration/oauth_provider.md#instance-wide-applications) with the `api` scope. 1. Set the value of your `redirect-uri` to the `pages-domain` authorization endpoint - - `http://pages.gdk.test:3010/auth`, for example - - The `redirect-uri` must not contain any GitLab Pages site domain. +(for example, `http://pages.gdk.test:3010/auth`). +The `redirect-uri` must not contain any GitLab Pages site domain. + 1. Add the auth client configuration: - With GDK, in `gdk.yml`: @@ -236,3 +238,29 @@ make acceptance # so we want to have the latest changes in the build that is tested make && go test ./ -run TestRedirect ``` + +## Contributing + +### Feature flags + +WARNING: +All newly-introduced feature flags should be [disabled by default](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#feature-flags-in-gitlab-development). + +Consider adding a [feature flag](../feature_flags/index.md) for any non-trivial changes. +Feature flags can make the release and rollback of these changes easier, avoiding +incidents and downtime. To add a new feature flag to GitLab Pages: + +1. Create the feature flag in + [`internal/feature/feature.go`](https://gitlab.com/gitlab-org/gitlab-pages/-/blob/master/internal/feature/feature.go), + which must be **off** by default. +1. Create an issue to track the feature flag using the `Feature Flag` template. +1. Add the `~"feature flag"` label to any merge requests that handle feature flags. + +For GitLab Pages, the feature flags are controlled by environment variables at a global level. +A deployment at the service level is required to change the state of a feature flag. +Example of a merge request enabling a GitLab Pages feature flag: +[Enforce GitLab Pages rate limits](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/merge_requests/1500) + +## Related topics + +- [Feature flags in the development of GitLab](../feature_flags/index.md) diff --git a/doc/development/performance.md b/doc/development/performance.md index d7cbef0a211..479782e0ccf 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -18,14 +18,13 @@ consistent performance of GitLab. Refer to the [Index](#performance-documentatio - Backend: - [Tooling](#tooling) - Database: - - [Query performance guidelines](../development/query_performance.md) + - [Query performance guidelines](database/query_performance.md) - [Pagination performance guidelines](../development/database/pagination_performance_guidelines.md) - [Keyset pagination performance](../development/database/keyset_pagination.md#performance) - [Troubleshooting import/export performance issues](../development/import_export.md#troubleshooting-performance-issues) - [Pipelines performance in the `gitlab` project](../development/pipelines.md#performance) - Frontend: - - [Performance guidelines](../development/fe_guide/performance.md) - - [Performance dashboards and monitoring guidelines](../development/new_fe_guide/development/performance.md) + - [Performance guidelines and monitoring](../development/fe_guide/performance.md) - [Browser performance testing guidelines](../ci/testing/browser_performance_testing.md) - [`gdk measure` and `gdk measure-workflow`](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/gdk_commands.md#measure-performance) - QA: @@ -927,12 +926,11 @@ SOME_CONSTANT = 'bar' You might want millions of project rows in your local database, for example, in order to compare relative query performance, or to reproduce a bug. You could -do this by hand with SQL commands or using [Mass Inserting Rails -Models](mass_insert.md) functionality. +do this by hand with SQL commands or using [Mass Inserting Rails Models](mass_insert.md) functionality. Assuming you are working with ActiveRecord models, you might also find these links helpful: -- [Insert records in batches](insert_into_tables_in_batches.md) +- [Insert records in batches](database/insert_into_tables_in_batches.md) - [BulkInsert gem](https://github.com/jamis/bulk_insert) - [ActiveRecord::PgGenerateSeries gem](https://github.com/ryu39/active_record-pg_generate_series) diff --git a/doc/development/permissions.md b/doc/development/permissions.md index ed95456c4f9..8e517b8577c 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -87,7 +87,7 @@ module): - Owner (`50`) If a user is the member of both a project and the project parent groups, the -higher permission is taken into account for the project. +highest permission is the applied access level for the project. If a user is the member of a project, but not the parent groups, they can still view the groups and their entities (like epics). diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index 2bf1e5a315a..d57e5bbeb26 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -221,8 +221,9 @@ that includes `rspec-profile` in their name. ### Logging -- Rails logging to `log/test.log` is disabled by default in CI [for - performance reasons](https://jtway.co/speed-up-your-rails-test-suite-by-6-in-1-line-13fedb869ec4). To override this setting, provide the +- Rails logging to `log/test.log` is disabled by default in CI + [for performance reasons](https://jtway.co/speed-up-your-rails-test-suite-by-6-in-1-line-13fedb869ec4). + To override this setting, provide the `RAILS_ENABLE_TEST_LOG` environment variable. ## Review app jobs @@ -247,6 +248,9 @@ The intent is to ensure that a change doesn't introduce a failure after `gitlab- ## As-if-JH jobs +NOTE: +This is disabled for now. + The `* as-if-jh` jobs run the GitLab test suite "as if JiHu", meaning as if the jobs would run in the context of [GitLab JH](jh_features_review.md). These jobs are only created in the following cases: @@ -261,12 +265,18 @@ The intent is to ensure that a change doesn't introduce a failure after `gitlab- ### When to consider applying `pipeline:run-as-if-jh` label +NOTE: +This is disabled for now. + If a Ruby file is renamed and there's a corresponding [`prepend_mod` line](jh_features_review.md#jh-features-based-on-ce-or-ee-features), it's likely that GitLab JH is relying on it and requires a corresponding change to rename the module or class it's prepending. ### Corresponding JH branch +NOTE: +This is disabled for now. + You can create a corresponding JH branch on [GitLab JH](https://jihulab.com/gitlab-cn/gitlab) by appending `-jh` to the branch name. If a corresponding JH branch is found, `* as-if-jh` jobs grab the `jh` folder from the respective branch, diff --git a/doc/development/policies.md b/doc/development/policies.md index c9e4fdb4350..f0c9d0ec5f9 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -74,8 +74,7 @@ Do not use boolean operators such as `&&` and `||` within the rule DSL, as conditions within rule blocks are objects, not booleans. The same applies for ternary operators (`condition ? ... : ...`), and `if` blocks. These operators cannot be overridden, and are hence banned via a -[custom -cop](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49771). +[custom cop](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49771). ## Scores, Order, Performance diff --git a/doc/development/polymorphic_associations.md b/doc/development/polymorphic_associations.md index bbeaab40a90..6b9158b8408 100644 --- a/doc/development/polymorphic_associations.md +++ b/doc/development/polymorphic_associations.md @@ -1,152 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/polymorphic_associations.md' +remove_date: '2022-11-04' --- -# Polymorphic Associations +This document was moved to [another location](database/polymorphic_associations.md). -**Summary:** always use separate tables instead of polymorphic associations. - -Rails makes it possible to define so called "polymorphic associations". This -usually works by adding two columns to a table: a target type column, and a -target ID. For example, at the time of writing we have such a setup for -`members` with the following columns: - -- `source_type`: a string defining the model to use, can be either `Project` or - `Namespace`. -- `source_id`: the ID of the row to retrieve based on `source_type`. For - example, when `source_type` is `Project` then `source_id` contains a - project ID. - -While such a setup may appear to be useful, it comes with many drawbacks; enough -that you should avoid this at all costs. - -## Space Wasted - -Because this setup relies on string values to determine the model to use, it -wastes a lot of space. For example, for `Project` and `Namespace` the -maximum size is 9 bytes, plus 1 extra byte for every string when using -PostgreSQL. While this may only be 10 bytes per row, given enough tables and -rows using such a setup we can end up wasting quite a bit of disk space and -memory (for any indexes). - -## Indexes - -Because our associations are broken up into two columns this may result in -requiring composite indexes for queries to be performed efficiently. While -composite indexes are not wrong at all, they can be tricky to set up as the -ordering of columns in these indexes is important to ensure optimal performance. - -## Consistency - -One really big problem with polymorphic associations is being unable to enforce -data consistency on the database level using foreign keys. For consistency to be -enforced on the database level one would have to write their own foreign key -logic to support polymorphic associations. - -Enforcing consistency on the database level is absolutely crucial for -maintaining a healthy environment, and thus is another reason to avoid -polymorphic associations. - -## Query Overhead - -When using polymorphic associations you always need to filter using both -columns. For example, you may end up writing a query like this: - -```sql -SELECT * -FROM members -WHERE source_type = 'Project' -AND source_id = 13083; -``` - -Here PostgreSQL can perform the query quite efficiently if both columns are -indexed. As the query gets more complex, it may not be able to use these -indexes effectively. - -## Mixed Responsibilities - -Similar to functions and classes, a table should have a single responsibility: -storing data with a certain set of pre-defined columns. When using polymorphic -associations, you are storing different types of data (possibly with -different columns set) in the same table. - -## The Solution - -Fortunately, there is a solution to these problems: use a -separate table for every type you would otherwise store in the same table. Using -a separate table allows you to use everything a database may provide to ensure -consistency and query data efficiently, without any additional application logic -being necessary. - -Let's say you have a `members` table storing both approved and pending members, -for both projects and groups, and the pending state is determined by the column -`requested_at` being set or not. Schema wise such a setup can lead to various -columns only being set for certain rows, wasting space. It's also possible that -certain indexes are only set for certain rows, again wasting space. Finally, -querying such a table requires less than ideal queries. For example: - -```sql -SELECT * -FROM members -WHERE requested_at IS NULL -AND source_type = 'GroupMember' -AND source_id = 4 -``` - -Instead such a table should be broken up into separate tables. For example, you -may end up with 4 tables in this case: - -- project_members -- group_members -- pending_project_members -- pending_group_members - -This makes querying data trivial. For example, to get the members of a group -you'd run: - -```sql -SELECT * -FROM group_members -WHERE group_id = 4 -``` - -To get all the pending members of a group in turn you'd run: - -```sql -SELECT * -FROM pending_group_members -WHERE group_id = 4 -``` - -If you want to get both you can use a `UNION`, though you need to be explicit -about what columns you want to `SELECT` as otherwise the result set uses the -columns of the first query. For example: - -```sql -SELECT id, 'Group' AS target_type, group_id AS target_id -FROM group_members - -UNION ALL - -SELECT id, 'Project' AS target_type, project_id AS target_id -FROM project_members -``` - -The above example is perhaps a bit silly, but it shows that there's nothing -stopping you from merging the data together and presenting it on the same page. -Selecting columns explicitly can also speed up queries as the database has to do -less work to get the data (compared to selecting all columns, even ones you're -not using). - -Our schema also becomes easier. No longer do we need to both store and index the -`source_type` column, we can define foreign keys easily, and we don't need to -filter rows using the `IS NULL` condition. - -To summarize: using separate tables allows us to use foreign keys effectively, -create indexes only where necessary, conserve space, query data more -efficiently, and scale these tables more easily (for example, by storing them on -separate disks). A nice side effect of this is that code can also become easier, -as a single model isn't responsible for handling different kinds of -data. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_count_limits.md b/doc/development/query_count_limits.md index 49509727337..f16c8cfc6cd 100644 --- a/doc/development/query_count_limits.md +++ b/doc/development/query_count_limits.md @@ -1,70 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/query_count_limits.md' +remove_date: '2022-11-06' --- -# Query Count Limits +This document was moved to [another location](database/query_count_limits.md). -Each controller or API endpoint is allowed to execute up to 100 SQL queries and -in test environments we raise an error when this threshold is exceeded. - -## Solving Failing Tests - -When a test fails because it executes more than 100 SQL queries there are two -solutions to this problem: - -- Reduce the number of SQL queries that are executed. -- Disable query limiting for the controller or API endpoint. - -You should only resort to disabling query limits when an existing controller or endpoint -is to blame as in this case reducing the number of SQL queries can take a lot of -effort. Newly added controllers and endpoints are not allowed to execute more -than 100 SQL queries and no exceptions are made for this rule. _If_ a large -number of SQL queries is necessary to perform certain work it's best to have -this work performed by Sidekiq instead of doing this directly in a web request. - -## Disable query limiting - -In the event that you _have_ to disable query limits for a controller, you must first -create an issue. This issue should (preferably in the title) mention the -controller or endpoint and include the appropriate labels (`database`, -`performance`, and at least a team specific label such as `Discussion`). - -After the issue has been created, you can disable query limits on the code in question. For -Rails controllers it's best to create a `before_action` hook that runs as early -as possible. The called method in turn should call -`Gitlab::QueryLimiting.disable!('issue URL here')`. For example: - -```ruby -class MyController < ApplicationController - before_action :disable_query_limiting, only: [:show] - - def index - # ... - end - - def show - # ... - end - - def disable_query_limiting - Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/...') - end -end -``` - -By using a `before_action` you don't have to modify the controller method in -question, reducing the likelihood of merge conflicts. - -For Grape API endpoints there unfortunately is not a reliable way of running a -hook before a specific endpoint. This means that you have to add the allowlist -call directly into the endpoint like so: - -```ruby -get '/projects/:id/foo' do - Gitlab::QueryLimiting.disable!('...') - - # ... -end -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md index 4fe27d42c38..618d007f766 100644 --- a/doc/development/query_performance.md +++ b/doc/development/query_performance.md @@ -1,74 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/query_performance.md' +remove_date: '2022-11-06' --- -# Query performance guidelines +This document was moved to [another location](database/query_performance.md). -This document describes various guidelines to follow when optimizing SQL queries. - -When you are optimizing your SQL queries, there are two dimensions to pay attention to: - -1. The query execution time. This is paramount as it reflects how the user experiences GitLab. -1. The query plan. Optimizing the query plan is important in allowing queries to independently scale over time. Realizing that an index keeps a query performing well as the table grows before the query degrades is an example of why we analyze these plans. - -## Timing guidelines for queries - -| Query Type | Maximum Query Time | Notes | -|----|----|---| -| General queries | `100ms` | This is not a hard limit, but if a query is getting above it, it is important to spend time understanding why it can or cannot be optimized. | -| Queries in a migration | `100ms` | This is different than the total [migration time](migration_style_guide.md#how-long-a-migration-should-take). | -| Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | -| Background migrations | `1s` | | -| Service Ping | `1s` | See the [Service Ping docs](service_ping/implement.md) for more details. | - -- When analyzing your query's performance, pay attention to if the time you are seeing is on a [cold or warm cache](#cold-and-warm-cache). These guidelines apply for both cache types. -- When working with batched queries, change the range and batch size to see how it effects the query timing and caching. -- If an existing query is not performing well, make an effort to improve it. If it is too complex or would stall development, create a follow-up so it can be addressed in a timely manner. You can always ask the database reviewer or maintainer for help and guidance. - -## Cold and warm cache - -When evaluating query performance it is important to understand the difference between -cold and warm cached queries. - -The first time a query is made, it is made on a "cold cache". Meaning it needs -to read from disk. If you run the query again, the data can be read from the -cache, or what PostgreSQL calls shared buffers. This is the "warm cache" query. - -When analyzing an [`EXPLAIN` plan](understanding_explain_plans.md), you can see -the difference not only in the timing, but by looking at the output for `Buffers` -by running your explain with `EXPLAIN(analyze, buffers)`. [Database Lab](understanding_explain_plans.md#database-lab-engine) -automatically includes these options. - -If you are making a warm cache query, you see only the `shared hits`. - -For example in #database-lab: - -```plaintext -Shared buffers: - - hits: 36467 (~284.90 MiB) from the buffer pool - - reads: 0 from the OS file cache, including disk I/O -``` - -Or in the explain plan from `psql`: - -```sql -Buffers: shared hit=7323 -``` - -If the cache is cold, you also see `reads`. - -In #database-lab: - -```plaintext -Shared buffers: - - hits: 17204 (~134.40 MiB) from the buffer pool - - reads: 15229 (~119.00 MiB) from the OS file cache, including disk I/O -``` - -In `psql`: - -```sql -Buffers: shared hit=7202 read=121 -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/query_recorder.md b/doc/development/query_recorder.md index 371d6e0e49e..cb05bc604af 100644 --- a/doc/development/query_recorder.md +++ b/doc/development/query_recorder.md @@ -1,145 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/query_recorder.md' +remove_date: '2022-11-06' --- -# QueryRecorder +This document was moved to [another location](database/query_recorder.md). -QueryRecorder is a tool for detecting the [N+1 queries problem](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations) from tests. - -> Implemented in [spec/support/query_recorder.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/helpers/query_recorder.rb) via [9c623e3e](https://gitlab.com/gitlab-org/gitlab-foss/commit/9c623e3e5d7434f2e30f7c389d13e5af4ede770a) - -As a rule, merge requests [should not increase query counts](merge_request_performance_guidelines.md#query-counts). If you find yourself adding something like `.includes(:author, :assignee)` to avoid having `N+1` queries, consider using QueryRecorder to enforce this with a test. Without this, a new feature which causes an additional model to be accessed can silently reintroduce the problem. - -## How it works - -This style of test works by counting the number of SQL queries executed by ActiveRecord. First a control count is taken, then you add new records to the database and rerun the count. If the number of queries has significantly increased then an `N+1` queries problem exists. - -```ruby -it "avoids N+1 database queries" do - control = ActiveRecord::QueryRecorder.new { visit_some_page } - create_list(:issue, 5) - expect { visit_some_page }.not_to exceed_query_limit(control) -end -``` - -You can if you wish, have both the expectation and the control as -`QueryRecorder` instances: - -```ruby -it "avoids N+1 database queries" do - control = ActiveRecord::QueryRecorder.new { visit_some_page } - create_list(:issue, 5) - action = ActiveRecord::QueryRecorder.new { visit_some_page } - - expect(action).not_to exceed_query_limit(control) -end -``` - -As an example you might create 5 issues in between counts, which would cause the query count to increase by 5 if an N+1 problem exists. - -In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible. - -If this test fails, and the control was passed as a `QueryRecorder`, then the -failure message indicates where the extra queries are by matching queries on -the longest common prefix, grouping similar queries together. - -## Cached queries - -By default, QueryRecorder ignores [cached queries](merge_request_performance_guidelines.md#cached-queries) in the count. However, it may be better to count -all queries to avoid introducing an N+1 query that may be masked by the statement cache. -To do this, this requires the `:use_sql_query_cache` flag to be set. -You should pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher: - -```ruby -it "avoids N+1 database queries", :use_sql_query_cache do - control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } - create_list(:issue, 5) - expect { visit_some_page }.not_to exceed_all_query_limit(control) -end -``` - -## Use request specs instead of controller specs - -Use a [request spec](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/spec/requests) when writing a N+1 test on the controller level. - -Controller specs should not be used to write N+1 tests as the controller is only initialized once per example. -This could lead to false successes where subsequent "requests" could have queries reduced (for example, because of memoization). - -## Finding the source of the query - -There are multiple ways to find the source of queries. - -- Inspect the `QueryRecorder` `data` attribute. It stores queries by `file_name:line_number:method_name`. - Each entry is a `hash` with the following fields: - - - `count`: the number of times a query from this `file_name:line_number:method_name` was called - - `occurrences`: the actual `SQL` of each call - - `backtrace`: the stack trace of each call (if either of the two following options were enabled) - - `QueryRecorder#find_query` allows filtering queries by their `file_name:line_number:method_name` and - `count` attributes. For example: - - ```ruby - control = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page } - control.find_query(/.*note.rb.*/, 0, first_only: true) - ``` - - `QueryRecorder#occurrences_by_line_method` returns a sorted array based on `data`, sorted by `count`. - -- View the call backtrace for the specific `QueryRecorder` instance you want - by using `ActiveRecord::QueryRecorder.new(query_recorder_debug: true)`. The output - is stored in file `test.log`. - -- Enable the call backtrace for all tests using the `QUERY_RECORDER_DEBUG` environment variable. - - To enable this, run the specs with the `QUERY_RECORDER_DEBUG` environment variable set. For example: - - ```shell - QUERY_RECORDER_DEBUG=1 bundle exec rspec spec/requests/api/projects_spec.rb - ``` - - This logs calls to QueryRecorder into the `test.log` file. For example: - - ```sql - QueryRecorder SQL: SELECT COUNT(*) FROM "issues" WHERE "issues"."deleted_at" IS NULL AND "issues"."project_id" = $1 AND ("issues"."state" IN ('opened')) AND "issues"."confidential" = $2 - --> /home/user/gitlab/gdk/gitlab/spec/support/query_recorder.rb:19:in `callback' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:127:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `block in finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `each' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/fanout.rb:46:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:36:in `finish' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications/instrumenter.rb:25:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract_adapter.rb:478:in `log' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:601:in `exec_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql_adapter.rb:585:in `execute_and_clear' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/postgresql/database_statements.rb:160:in `exec_query' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:356:in `select' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/database_statements.rb:32:in `select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `block in select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:83:in `cache_sql' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/connection_adapters/abstract/query_cache.rb:68:in `select_all' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:270:in `execute_simple_calculation' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:227:in `perform_calculation' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:133:in `calculate' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activerecord-4.2.8/lib/active_record/relation/calculations.rb:48:in `count' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:20:in `uncached_count' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `block in count' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `block in fetch' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:585:in `block in save_block_result_to_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `block in instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/notifications.rb:166:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:547:in `instrument' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:584:in `save_block_result_to_cache' - --> /home/user/.rbenv/versions/2.3.5/lib/ruby/gems/2.3.0/gems/activesupport-4.2.8/lib/active_support/cache.rb:299:in `fetch' - --> /home/user/gitlab/gdk/gitlab/app/services/base_count_service.rb:12:in `count' - --> /home/user/gitlab/gdk/gitlab/app/models/project.rb:1296:in `open_issues_count' - ``` - -## See also - -- [Bullet](profiling.md#bullet) For finding `N+1` query problems -- [Performance guidelines](performance.md) -- [Merge request performance guidelines - Query counts](merge_request_performance_guidelines.md#query-counts) -- [Merge request performance guidelines - Cached queries](merge_request_performance_guidelines.md#cached-queries) +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/rails_update.md b/doc/development/rails_update.md index 36ffae97377..9907a78421f 100644 --- a/doc/development/rails_update.md +++ b/doc/development/rails_update.md @@ -27,8 +27,8 @@ We strive to run GitLab using the latest Rails releases to benefit from performa 1. Run `yarn patch-package @rails/ujs` after updating this to ensure our local patch file version matches. 1. Create an MR with the `pipeline:run-all-rspec` label and see if pipeline breaks. 1. To resolve and debug spec failures use `git bisect` against the rails repository. See the [debugging section](#git-bisect-against-rails) below. -1. Include links to the Gem diffs between the two versions in the merge request description. For example, this is the gem diff for [`activesupport` 6.1.3.2 to -6.1.4.1](https://my.diffend.io/gems/activerecord/6.1.3.2/6.1.4.1). +1. Include links to the Gem diffs between the two versions in the merge request description. For example, this is the gem diff for + [`activesupport` 6.1.3.2 to 6.1.4.1](https://my.diffend.io/gems/activerecord/6.1.3.2/6.1.4.1). ### Prepare an MR for Gitaly diff --git a/doc/development/real_time.md b/doc/development/real_time.md index df725a36a93..21f3ee1f3b2 100644 --- a/doc/development/real_time.md +++ b/doc/development/real_time.md @@ -60,8 +60,8 @@ downstream services. To mitigate this, ensure that the code establishing the new WebSocket connection is feature flagged and defaulted to `off`. A careful, percentage-based roll-out -of the feature flag ensures that effects can be observed on the [WebSocket -dashboard](https://dashboards.gitlab.net/d/websockets-main/websockets-overview?orgId=1) +of the feature flag ensures that effects can be observed on the +[WebSocket dashboard](https://dashboards.gitlab.net/d/websockets-main/websockets-overview?orgId=1) 1. Create a [feature flag roll-out](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md) diff --git a/doc/development/redis/new_redis_instance.md b/doc/development/redis/new_redis_instance.md index 4900755b58c..efaf1e5a6d0 100644 --- a/doc/development/redis/new_redis_instance.md +++ b/doc/development/redis/new_redis_instance.md @@ -169,7 +169,7 @@ MultiStore uses two feature flags to control the actual migration: - `use_primary_and_secondary_stores_for_[store_name]` - `use_primary_store_as_default_for_[store_name]` -For example, if our new Redis instance is called `Gitlab::Redis::Foo`, we can [create](../../../ee/development/feature_flags/#create-a-new-feature-flag) two feature flags by executing: +For example, if our new Redis instance is called `Gitlab::Redis::Foo`, we can [create](../feature_flags/index.md#create-a-new-feature-flag) two feature flags by executing: ```shell bin/feature-flag use_primary_and_secondary_stores_for_foo @@ -265,7 +265,7 @@ instances to cope without this functional partition. If we decide to keep the migration code: - We should document the migration steps. -- If we used a feature flag, we should ensure it's an [ops type feature - flag](../feature_flags/index.md#ops-type), as these are long-lived flags. +- If we used a feature flag, we should ensure it's an + [ops type feature flag](../feature_flags/index.md#ops-type), as these are long-lived flags. Otherwise, we can remove the flags and conclude the project. diff --git a/doc/development/reusing_abstractions.md b/doc/development/reusing_abstractions.md index f3eb1ebcc0c..ef4e8b0310f 100644 --- a/doc/development/reusing_abstractions.md +++ b/doc/development/reusing_abstractions.md @@ -109,7 +109,7 @@ the various abstractions and what they can (not) reuse: | Abstraction | Service classes | Finders | Presenters | Serializers | Model instance method | Model class methods | Active Record | Worker |:-----------------------|:-----------------|:---------|:------------|:--------------|:------------------------|:----------------------|:----------------|:-------- -| Controller | Yes | Yes | Yes | Yes | Yes | No | No | No +| Controller/API endpoint| Yes | Yes | Yes | Yes | Yes | No | No | No | Service class | Yes | Yes | No | No | Yes | No | No | Yes | Finder | No | No | No | No | Yes | Yes | No | No | Presenter | No | Yes | No | No | Yes | Yes | No | No @@ -125,9 +125,11 @@ Everything in `app/controllers`. Controllers should not do much work on their own, instead they simply pass input to other classes and present the results. -### Grape endpoint +### API endpoints -Everything in `lib/api`. +Everything in `lib/api` (the REST API) and `app/graphql` (the GraphQL API). + +API endpoints have the same abstraction level as controllers. ### Service classes @@ -145,6 +147,27 @@ Legacy classes inherited from `BaseService` for historical reasons. In Service classes the use of `execute` and `#execute` is preferred over `call` and `#call`. +Model properties should be passed to the constructor in a `params` hash, and will be assigned directly. + +To pass extra parameters (which need to be processed, and are not model properties), +include an `options` hash in the constructor and store it in an instance variable: + +```ruby +# container: Project, or Group +# current_user: Current user +# params: Model properties from the controller, already allowlisted with strong parameters +# options: Configuration for this service, can be any of the following: +# notify: Whether to send a notifcation to the current user +# cc: Email address to copy when sending a notification +def initialize(container:, current_user: nil, params: {}, options: {}) + super(container, current_user, params) + @options = options +end +``` + +View the [initial discussion](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90008#note_988744060) +and [further discussion](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90853#note_1053425083). + Classes that are not service objects should be [created elsewhere](directory_structure.md#use-namespaces-to-define-bounded-contexts), such as in `lib`. #### ServiceResponse @@ -206,7 +229,27 @@ See [the documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/p Everything in `app/serializers`, used for presenting the response to a request, typically in JSON. -### Model class methods +### Models + +Classes and modules in `app/models` represent domain concepts that encapsulate both +[data and behavior](https://en.wikipedia.org/wiki/Domain_model). + +These classes can interact directly with a data store (like ActiveRecord models) or +can be a thin wrapper (Plain Old Ruby Objects) on top of ActiveRecord models to express a +richer domain concept. + +[Entities and Value Objects](https://martinfowler.com/bliki/EvansClassification.html) +that represent domain concepts are considered domain models. + +Some examples: + +- [`DesignManagement::DesignAtVersion`](https://gitlab.com/gitlab-org/gitlab/-/blob/b62ce98cff8e0530210670f9cb0314221181b77f/app/models/design_management/design_at_version.rb) + is a model that leverages validations to combine designs and versions. +- [`Ci::Minutes::Usage`](https://gitlab.com/gitlab-org/gitlab/-/blob/ec52f19f7325410177c00fef06379f55ab7cab67/ee/app/models/ci/minutes/usage.rb) + is a Value Object that provides [CI/CD minutes usage](../ci/pipelines/cicd_minutes.md) + for a given namespace. + +#### Model class methods These are class methods defined by _GitLab itself_, including the following methods provided by Active Record: @@ -220,7 +263,7 @@ methods provided by Active Record: Any other methods such as `find_by(some_column: X)` are not included, and instead fall under the "Active Record" abstraction. -### Model instance methods +#### Model instance methods Instance methods defined on Active Record models by _GitLab itself_. Methods provided by Active Record are not included, except for the following methods: @@ -230,7 +273,7 @@ provided by Active Record are not included, except for the following methods: - `destroy` - `delete` -### Active Record +#### Active Record The API provided by Active Record itself, such as the `where` method, `save`, `delete_all`, and so on. diff --git a/doc/development/routing.md b/doc/development/routing.md index 2b3ecd8127b..3d5857b4237 100644 --- a/doc/development/routing.md +++ b/doc/development/routing.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Routing -The GitLab backend is written primarily with Rails so it uses [Rails -routing](https://guides.rubyonrails.org/routing.html). Beside Rails best +The GitLab backend is written primarily with Rails so it uses +[Rails routing](https://guides.rubyonrails.org/routing.html). Beside Rails best practices, there are few rules unique to the GitLab application. To support subgroups, GitLab project and group routes use the wildcard character to match project and group routes. For example, we might have diff --git a/doc/development/scalability.md b/doc/development/scalability.md index 39cd0ecfcdd..b7ee0ca1167 100644 --- a/doc/development/scalability.md +++ b/doc/development/scalability.md @@ -35,8 +35,8 @@ The application has a tight coupling to the database schema. When the application starts, Rails queries the database schema, caching the tables and column types for the data requested. Because of this schema cache, dropping a column or table while the application is running can produce 500 errors to the -user. This is why we have a [process for dropping columns and other -no-downtime changes](database/avoiding_downtime_in_migrations.md). +user. This is why we have a +[process for dropping columns and other no-downtime changes](database/avoiding_downtime_in_migrations.md). #### Multi-tenancy @@ -61,11 +61,11 @@ There are two ways to deal with this: - Sharding. Distribute data across multiple databases. Partitioning is a built-in PostgreSQL feature and requires minimal changes -in the application. However, it [requires PostgreSQL -11](https://www.2ndquadrant.com/en/blog/partitioning-evolution-postgresql-11/). +in the application. However, it +[requires PostgreSQL 11](https://www.2ndquadrant.com/en/blog/partitioning-evolution-postgresql-11/). -For example, a natural way to partition is to [partition tables by -dates](https://gitlab.com/groups/gitlab-org/-/epics/2023). For example, +For example, a natural way to partition is to +[partition tables by dates](https://gitlab.com/groups/gitlab-org/-/epics/2023). For example, the `events` and `audit_events` table are natural candidates for this kind of partitioning. @@ -77,10 +77,10 @@ to abstract data access into API calls that abstract the database from the application, but this is a significant amount of work. There are solutions that may help abstract the sharding to some extent -from the application. For example, we want to look at [Citus -Data](https://www.citusdata.com/product/community) closely. Citus Data -provides a Rails plugin that adds a [tenant ID to ActiveRecord -models](https://www.citusdata.com/blog/2017/01/05/easily-scale-out-multi-tenant-apps/). +from the application. For example, we want to look at +[Citus Data](https://www.citusdata.com/product/community) closely. Citus Data +provides a Rails plugin that adds a +[tenant ID to ActiveRecord models](https://www.citusdata.com/blog/2017/01/05/easily-scale-out-multi-tenant-apps/). Sharding can also be done based on feature verticals. This is the microservice approach to sharding, where each service represents a @@ -97,12 +97,12 @@ systems. #### Database size -A recent [database checkup shows a breakdown of the table sizes on -GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/8022#master-1022016101-8). +A recent +[database checkup shows a breakdown of the table sizes on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/8022#master-1022016101-8). Since `merge_request_diff_files` contains over 1 TB of data, we want to -reduce/eliminate this table first. GitLab has support for [storing diffs in -object storage](../administration/merge_request_diffs.md), which we [want to do on -GitLab.com](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7356). +reduce/eliminate this table first. GitLab has support for +[storing diffs in object storage](../administration/merge_request_diffs.md), which we +[want to do on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/7356). #### High availability @@ -128,8 +128,7 @@ some actions that aren't traditionally available in standard load balancers. For example, the application considers a replica only if its replication lag is low (for example, WAL data behind by less than 100 MB). -More [details are in a blog -post](https://about.gitlab.com/blog/2017/10/02/scaling-the-gitlab-database/). +More [details are in a blog post](https://about.gitlab.com/blog/2017/10/02/scaling-the-gitlab-database/). ### PgBouncer @@ -148,10 +147,10 @@ limitation: - Run multiple PgBouncer instances. - Use a multi-threaded connection pooler (for example, - [Odyssey](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7776). + [Odyssey](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/7776). -On some Linux systems, it's possible to run [multiple PgBouncer instances on -the same port](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4796). +On some Linux systems, it's possible to run +[multiple PgBouncer instances on the same port](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4796). On GitLab.com, we run multiple PgBouncer instances on different ports to avoid saturating a single core. diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 9048da77071..8053b4285e6 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -196,7 +196,7 @@ Go's [`regexp`](https://pkg.go.dev/regexp) package uses `re2` and isn't vulnerab - [Rubular](https://rubular.com/) is a nice online tool to fiddle with Ruby Regexps. - [Runaway Regular Expressions](https://www.regular-expressions.info/catastrophic.html) -- [The impact of regular expression denial of service (ReDoS) in practice: an empirical study at the ecosystem scale](https://people.cs.vt.edu/~davisjam/downloads/publications/DavisCoghlanServantLee-EcosystemREDOS-ESECFSE18.pdf). This research paper discusses approaches to automatically detect ReDoS vulnerabilities. +- [The impact of regular expression denial of service (ReDoS) in practice: an empirical study at the ecosystem scale](https://davisjam.github.io/files/publications/DavisCoghlanServantLee-EcosystemREDOS-ESECFSE18.pdf). This research paper discusses approaches to automatically detect ReDoS vulnerabilities. - [Freezing the web: A study of ReDoS vulnerabilities in JavaScript-based web servers](https://www.usenix.org/system/files/conference/usenixsecurity18/sec18-staicu.pdf). Another research paper about detecting ReDoS vulnerabilities. ## Server Side Request Forgery (SSRF) diff --git a/doc/development/serializing_data.md b/doc/development/serializing_data.md index 97e6f665484..aa8b20eded7 100644 --- a/doc/development/serializing_data.md +++ b/doc/development/serializing_data.md @@ -1,90 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/serializing_data.md' +remove_date: '2022-11-06' --- -# Serializing Data +This document was moved to [another location](database/serializing_data.md). -**Summary:** don't store serialized data in the database, use separate columns -and/or tables instead. This includes storing of comma separated values as a -string. - -Rails makes it possible to store serialized data in JSON, YAML or other formats. -Such a field can be defined as follows: - -```ruby -class Issue < ActiveRecord::Model - serialize :custom_fields -end -``` - -While it may be tempting to store serialized data in the database there are many -problems with this. This document outlines these problems and provide an -alternative. - -## Serialized Data Is Less Powerful - -When using a relational database you have the ability to query individual -fields, change the schema, index data, and so forth. When you use serialized data -all of that becomes either very difficult or downright impossible. While -PostgreSQL does offer the ability to query JSON fields it is mostly meant for -very specialized use cases, and not for more general use. If you use YAML in -turn there's no way to query the data at all. - -## Waste Of Space - -Storing serialized data such as JSON or YAML ends up wasting a lot of space. -This is because these formats often include additional characters (for example, double -quotes or newlines) besides the data that you are storing. - -## Difficult To Manage - -There comes a time where you must add a new field to the serialized -data, or change an existing one. Using serialized data this becomes difficult -and very time consuming as the only way of doing so is to re-write all the -stored values. To do so you would have to: - -1. Retrieve the data -1. Parse it into a Ruby structure -1. Mutate it -1. Serialize it back to a String -1. Store it in the database - -On the other hand, if one were to use regular columns adding a column would be: - -```sql -ALTER TABLE table_name ADD COLUMN column_name type; -``` - -Such a query would take very little to no time and would immediately apply to -all rows, without having to re-write large JSON or YAML structures. - -Finally, there comes a time when the JSON or YAML structure is no longer -sufficient and you must migrate away from it. When storing only a few rows -this may not be a problem, but when storing millions of rows such a migration -can take hours or even days to complete. - -## Relational Databases Are Not Document Stores - -When storing data as JSON or YAML you're essentially using your database as if -it were a document store (for example, MongoDB), except you're not using any of the -powerful features provided by a typical RDBMS _nor_ are you using any of the -features provided by a typical document store (for example, the ability to index fields -of documents with variable fields). In other words, it's a waste. - -## Consistent Fields - -One argument sometimes made in favour of serialized data is having to store -widely varying fields and values. Sometimes this is truly the case, and then -perhaps it might make sense to use serialized data. However, in 99% of the cases -the fields and types stored tend to be the same for every row. Even if there is -a slight difference you can still use separate columns and just not set the ones -you don't need. - -## The Solution - -The solution is to use separate columns and/or separate tables. -This allows you to use all the features provided by your database, it -makes it easier to manage and migrate the data, you conserve space, you can -index the data efficiently and so forth. +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/service_measurement.md b/doc/development/service_measurement.md index 82fb9731bcb..17e509672f2 100644 --- a/doc/development/service_measurement.md +++ b/doc/development/service_measurement.md @@ -19,7 +19,7 @@ The measuring module is a tool that allows to measure a service's execution, and - RSS memory usage - Server worker ID -The measuring module logs these measurements into a structured log called [`service_measurement.log`](../administration/logs.md#service_measurementlog), +The measuring module logs these measurements into a structured log called [`service_measurement.log`](../administration/logs/index.md#service_measurementlog), as a single entry for each service execution. For GitLab.com, `service_measurement.log` is ingested in Elasticsearch and Kibana as part of our monitoring solution. diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md index 3263ba6458e..0ebc58dd669 100644 --- a/doc/development/service_ping/implement.md +++ b/doc/development/service_ping/implement.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Service Ping consists of two kinds of data: - **Counters**: Track how often a certain event happened over time, such as how many CI/CD pipelines have run. - They are monotonic and always trend up. + They are monotonic and usually trend up. - **Observations**: Facts collected from one or more GitLab instances and can carry arbitrary data. There are no general guidelines for how to collect those, due to the individual nature of that data. @@ -94,7 +94,7 @@ add_metric('CountUsersAssociatingMilestonesToReleasesMetric', time_frame: 'all') ``` WARNING: -Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../iterating_tables_in_batches.md) guide. +Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../database/iterating_tables_in_batches.md) guide. Examples: @@ -269,9 +269,15 @@ Arguments: #### Ordinary Redis counters -Example of implementation: +Example of implementation: [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb), using Redis methods [`INCR`](https://redis.io/commands/incr) and [`GET`](https://redis.io/commands/get). -Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) +Events are handled by counter classes in the `Gitlab::UsageDataCounters` namespace, inheriting from `BaseCounter`, that are either: + +1. Listed in [`Gitlab::UsageDataCounters::COUNTERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters.rb#L5) to be then included in `Gitlab::UsageData`. + +1. Specified in the metric definition using the `RedisMetric` instrumentation class as a `counter_class` option to be picked up using the [metric instrumentation](metrics_instrumentation.md) framework. Refer to the [Redis metrics](metrics_instrumentation.md#redis-metrics) documentation for an example implementation. + +Inheriting classes are expected to override `KNOWN_EVENTS` and `PREFIX` constants to build event names and associated metrics. For example, for prefix `issues` and events array `%w[create, update, delete]`, three metrics will be added to the Service Ping payload: `counts.issues_create`, `counts.issues_update` and `counts.issues_delete`. ##### `UsageData` API @@ -316,7 +322,7 @@ Enabled by default in GitLab 13.7 and later. #### Redis HLL counters WARNING: -HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes some small error**. According to [Redis documentation](https://redis.io/commands/pfcount), data from +HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes some small error**. According to [Redis documentation](https://redis.io/commands/pfcount/), data from used HLL implementation is "approximated with a standard error of 0.81%". NOTE: @@ -324,7 +330,7 @@ NOTE: With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. -Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). +Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd/) and [PFCOUNT](https://redis.io/commands/pfcount/). ##### Add new events @@ -371,14 +377,15 @@ Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PF - In the controller using the `RedisTracking` module and the following format: ```ruby - track_redis_hll_event(*controller_actions, name:, if: nil, &block) + track_event(*controller_actions, name:, conditions: nil, destinations: [:redis_hll], &block) ``` Arguments: - `controller_actions`: the controller actions to track. - `name`: the event name. - - `if`: optional custom conditions. Uses the same format as Rails callbacks. + - `conditions`: optional custom conditions. Uses the same format as Rails callbacks. + - `destinations`: optional list of destinations. Currently supports `:redis_hll` and `:snowplow`. Default: [:redis_hll]. - `&block`: optional block that computes and returns the `custom_id` that we want to track. This overrides the `visitor_id`. Example: @@ -389,7 +396,7 @@ Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PF include RedisTracking skip_before_action :authenticate_user!, only: :show - track_redis_hll_event :index, :show, name: 'users_visiting_projects' + track_event :index, :show, name: 'users_visiting_projects' def index render html: 'index' @@ -688,7 +695,7 @@ pry(main)> Gitlab::UsageData.count(User.active) Paste the SQL query into `#database-lab` to see how the query performs at scale. - GitLab.com's production database has a 15 second timeout. -- Any single query must stay below the [1 second execution time](../query_performance.md#timing-guidelines-for-queries) with cold caches. +- Any single query must stay below the [1 second execution time](../database/query_performance.md#timing-guidelines-for-queries) with cold caches. - Add a specialized index on columns involved to reduce the execution time. To understand the query's execution, we add the following information diff --git a/doc/development/service_ping/index.md b/doc/development/service_ping/index.md index cd8af3e9152..4481fe33bda 100644 --- a/doc/development/service_ping/index.md +++ b/doc/development/service_ping/index.md @@ -22,9 +22,7 @@ and sales teams understand how GitLab is used. The data helps to: Service Ping information is not anonymous. It's linked to the instance's hostname, but does not contain project names, usernames, or any other specific data. -Sending a Service Ping payload is optional and you can [disable](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics) it on any -self-managed instance. When Service Ping is enabled, GitLab gathers data from the other instances -and can show your instance's usage statistics to your users. +Service Ping is enabled by default. However, you can [disable](../../user/admin_area/settings/usage_statistics.md#enable-or-disable-usage-statistics) it on any self-managed instance. When Service Ping is enabled, GitLab gathers data from the other instances and can show your instance's usage statistics to your users. ## Service Ping terminology @@ -113,7 +111,7 @@ sequenceDiagram 1. Finally, the timing metadata information that is used for diagnostic purposes is submitted to the Versions application. It consists of a list of metric identifiers and the time it took to calculate the metrics: - > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37911) in GitLab 15.0 [with a flag(../../user/feature_flags.md), enabled by default. + > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37911) in GitLab 15.0 [with a flag](../../user/feature_flags.md), enabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/295289) in GitLab 15.2. [Feature flag `measure_service_ping_metric_collection`](https://gitlab.com/gitlab-org/gitlab/-/issues/358128) removed. ```ruby diff --git a/doc/development/service_ping/metrics_dictionary.md b/doc/development/service_ping/metrics_dictionary.md index 2adba5d8095..d063c4c7601 100644 --- a/doc/development/service_ping/metrics_dictionary.md +++ b/doc/development/service_ping/metrics_dictionary.md @@ -205,8 +205,8 @@ instance unique identifier. key_path: uuid description: GitLab instance unique identifier product_category: collection -product_section: growth -product_stage: growth +product_section: analytics +product_stage: analytics product_group: product_intelligence value_type: string status: active @@ -301,7 +301,7 @@ bundle exec rails generate gitlab:usage_metric_definition:redis_hll issues users ## Metrics Dictionary -[Metrics Dictionary is a separate application](https://gitlab.com/gitlab-org/growth/product-intelligence/metric-dictionary). +[Metrics Dictionary is a separate application](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/metric-dictionary). All metrics available in Service Ping are in the [Metrics Dictionary](https://metrics.gitlab.com/). diff --git a/doc/development/service_ping/metrics_instrumentation.md b/doc/development/service_ping/metrics_instrumentation.md index e1c51713f3c..9dc37386111 100644 --- a/doc/development/service_ping/metrics_instrumentation.md +++ b/doc/development/service_ping/metrics_instrumentation.md @@ -29,7 +29,7 @@ A metric definition has the [`instrumentation_class`](metrics_dictionary.md) fie The defined instrumentation class should inherit one of the existing metric classes: `DatabaseMetric`, `RedisMetric`, `RedisHLLMetric`, `NumbersMetric` or `GenericMetric`. -The current convention is that a single instrumentation class corresponds to a single metric. On a rare occasions, there are exceptions to that convention like [Redis metrics](#redis-metrics). To use a single instrumentation class for more than one metric, please reach out to one of the `@gitlab-org/growth/product-intelligence/engineers` members to consult about your case. +The current convention is that a single instrumentation class corresponds to a single metric. On rare occasions, there are exceptions to that convention like [Redis metrics](#redis-metrics). To use a single instrumentation class for more than one metric, please reach out to one of the `@gitlab-org/analytics-section/product-intelligence/engineers` members to consult about your case. Using the instrumentation classes ensures that metrics can fail safe individually, without breaking the entire process of Service Ping generation. @@ -38,12 +38,15 @@ We have built a domain-specific language (DSL) to define the metrics instrumenta ## Database metrics +You can use database metrics to track data kept in the database, for example, a count of issues that exist on a given instance. + - `operation`: Operations for the given `relation`, one of `count`, `distinct_count`, `sum`, and `average`. - `relation`: `ActiveRecord::Relation` for the objects we want to perform the `operation`. - `start`: Specifies the start value of the batch counting, by default is `relation.minimum(:id)`. - `finish`: Specifies the end value of the batch counting, by default is `relation.maximum(:id)`. - `cache_start_and_finish_as`: Specifies the cache key for `start` and `finish` values and sets up caching them. Use this call when `start` and `finish` are expensive queries that should be reused between different metric calculations. - `available?`: Specifies whether the metric should be reported. The default is `true`. +- `timestamp_column`: Optionally specifies timestamp column for metric used to filter records for time constrained metrics. The default is `created_at`. [Example of a merge request that adds a database metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60022). @@ -149,6 +152,8 @@ end ## Redis metrics +You can use Redis metrics to track events not kept in the database, for example, a count of how many times the search bar has been used. + [Example of a merge request that adds a `Redis` metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66582). Count unique values for `source_code_pushes` event. @@ -199,6 +204,9 @@ options: ``` ## Redis HyperLogLog metrics + +You can use Redis HyperLogLog metrics to track events not kept in the database and incremented for unique values such as unique users, +for example, a count of how many different users used the search bar. [Example of a merge request that adds a `RedisHLL` metric](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61685). @@ -283,6 +291,8 @@ instrumentation_class: 'IssuesBoardsCountMetric' ## Generic metrics +You can use generic metrics for other metrics, for example, an instance's database version. Observations type of data will always have a Generic metric counter type. + - `value`: Specifies the value of the metric. - `available?`: Specifies whether the metric should be reported. The default is `true`. diff --git a/doc/development/service_ping/performance_indicator_metrics.md b/doc/development/service_ping/performance_indicator_metrics.md index bdd4c319d41..d2abc597a22 100644 --- a/doc/development/service_ping/performance_indicator_metrics.md +++ b/doc/development/service_ping/performance_indicator_metrics.md @@ -10,8 +10,7 @@ This guide describes how to use metrics definitions to define [performance indic To use a metric definition to manage a performance indicator: -1. Create a new issue and use the [Performance Indicator Metric issue template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Performance%20Indicator%20Metric). +1. Create a merge request that includes related changes. 1. Use labels `~"product intelligence"`, `"~Data Warehouse::Impact Check"`. -1. Create a merge request that includes changes related only to the metric performance indicator. 1. Update the metric definition `performance_indicator_type` [field](metrics_dictionary.md#metrics-definition-and-validation). -1. Create an issue in GitLab Data Team project with the [Product Performance Indicator template](https://gitlab.com/gitlab-data/analytics/-/issues/new?issuable_template=Product%20Performance%20Indicator%20Template). +1. Create an issue in GitLab Product Data Insights project with the [PI Chart Help template](https://gitlab.com/gitlab-data/product-analytics/-/issues/new?issuable_template=PI%20Chart%20Help) to have the new metric visualized. diff --git a/doc/development/service_ping/review_guidelines.md b/doc/development/service_ping/review_guidelines.md index 4ce5b2d577c..1b00858be7e 100644 --- a/doc/development/service_ping/review_guidelines.md +++ b/doc/development/service_ping/review_guidelines.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Service Ping review guidelines This page includes introductory material for a -[Product Intelligence](https://about.gitlab.com/handbook/engineering/development/growth/product-intelligence/) +[Product Intelligence](https://about.gitlab.com/handbook/engineering/development/analytics/product-intelligence/) review, and is specific to Service Ping related reviews. For broader advice and general best practices for code reviews, refer to our [code review guide](../code_review.md). @@ -42,7 +42,7 @@ are regular backend changes. - Assign both the `~backend` and `~product intelligence` reviews to another Product Intelligence team member. - Assign the maintainer review to someone outside of the Product Intelligence group. - Assign an - [engineer](https://gitlab.com/groups/gitlab-org/growth/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) from the Product Intelligence team for a review. + [engineer](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) from the Product Intelligence team for a review. - Set the correct attributes in the metric's YAML definition: - `product_section`, `product_stage`, `product_group`, `product_category` - Provide a clear description of the metric. @@ -76,7 +76,7 @@ are regular backend changes. [Danger bot](../dangerbot.md) adds the list of changed Product Intelligence files and pings the -[`@gitlab-org/growth/product-intelligence/engineers`](https://gitlab.com/groups/gitlab-org/growth/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) group for merge requests +[`@gitlab-org/analytics-section/product-intelligence/engineers`](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) group for merge requests that are not drafts. Any of the Product Intelligence engineers can be assigned for the Product Intelligence review. diff --git a/doc/development/service_ping/usage_data.md b/doc/development/service_ping/usage_data.md index a659bbf2265..4181bd90a02 100644 --- a/doc/development/service_ping/usage_data.md +++ b/doc/development/service_ping/usage_data.md @@ -59,7 +59,7 @@ Arguments: - `end`: custom end of the batch counting to avoid complex min calculations WARNING: -Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../iterating_tables_in_batches.md) guide. +Counting over non-unique columns can lead to performance issues. For more information, see the [iterating tables in batches](../database/iterating_tables_in_batches.md) guide. Examples: diff --git a/doc/development/sha1_as_binary.md b/doc/development/sha1_as_binary.md index a7bb3001ddb..7f928d09470 100644 --- a/doc/development/sha1_as_binary.md +++ b/doc/development/sha1_as_binary.md @@ -1,42 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/sha1_as_binary.md' +remove_date: '2022-11-06' --- -# Storing SHA1 Hashes As Binary +This document was moved to [another location](database/sha1_as_binary.md). -Storing SHA1 hashes as strings is not very space efficient. A SHA1 as a string -requires at least 40 bytes, an additional byte to store the encoding, and -perhaps more space depending on the internals of PostgreSQL. - -On the other hand, if one were to store a SHA1 as binary one would only need 20 -bytes for the actual SHA1, and 1 or 4 bytes of additional space (again depending -on database internals). This means that in the best case scenario we can reduce -the space usage by 50%. - -To make this easier to work with you can include the concern `ShaAttribute` into -a model and define a SHA attribute using the `sha_attribute` class method. For -example: - -```ruby -class Commit < ActiveRecord::Base - include ShaAttribute - - sha_attribute :sha -end -``` - -This allows you to use the value of the `sha` attribute as if it were a string, -while storing it as binary. This means that you can do something like this, -without having to worry about converting data to the right binary format: - -```ruby -commit = Commit.find_by(sha: '88c60307bd1f215095834f09a1a5cb18701ac8ad') -commit.sha = '971604de4cfa324d91c41650fabc129420c8d1cc' -commit.save -``` - -There is however one requirement: the column used to store the SHA has _must_ be -a binary type. For Rails this means you need to use the `:binary` type instead -of `:text` or `:string`. +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/shell_commands.md b/doc/development/shell_commands.md index fcb8c20bdd3..d8a3f86685e 100644 --- a/doc/development/shell_commands.md +++ b/doc/development/shell_commands.md @@ -17,7 +17,7 @@ These guidelines are meant to make your code more reliable _and_ secure. ## Use File and FileUtils instead of shell commands -Sometimes we invoke basic Unix commands via the shell when there is also a Ruby API for doing it. Use the Ruby API if it exists. <https://www.ruby-doc.org/stdlib-2.0.0/libdoc/fileutils/rdoc/FileUtils.html#module-FileUtils-label-Module+Functions> +Sometimes we invoke basic Unix commands via the shell when there is also a Ruby API for doing it. Use [the Ruby API](https://ruby-doc.org/stdlib-2.0.0/libdoc/fileutils/rdoc/FileUtils.html#module-FileUtils-label-Module+Functions) if it exists. ```ruby # Wrong diff --git a/doc/development/sidekiq/compatibility_across_updates.md b/doc/development/sidekiq/compatibility_across_updates.md index 96a3573d11a..1d369b5a970 100644 --- a/doc/development/sidekiq/compatibility_across_updates.md +++ b/doc/development/sidekiq/compatibility_across_updates.md @@ -18,18 +18,17 @@ several possible situations: ## Adding new workers -On GitLab.com, we [do not currently have a Sidekiq deployment in the -canary stage](https://gitlab.com/gitlab-org/gitlab/-/issues/19239). This -means that a new worker than can be scheduled from an HTTP endpoint may +On GitLab.com, we +[do not currently have a Sidekiq deployment in the canary stage](https://gitlab.com/gitlab-org/gitlab/-/issues/19239). +This means that a new worker than can be scheduled from an HTTP endpoint may be scheduled from canary but not run on Sidekiq until the full production deployment is complete. This can be several hours later than scheduling the job. For some workers, this will not be a problem. For -others - particularly [latency-sensitive -jobs](worker_attributes.md#latency-sensitive-jobs) - this will result in a poor user -experience. +others - particularly [latency-sensitive jobs](worker_attributes.md#latency-sensitive-jobs) - +this will result in a poor user experience. This only applies to new worker classes when they are first introduced. -As we recommend [using feature flags](../feature_flags/) as a general +As we recommend [using feature flags](../feature_flags/index.md) as a general development process, it's best to control the entire change (including scheduling of the new Sidekiq worker) with a feature flag. diff --git a/doc/development/sidekiq/idempotent_jobs.md b/doc/development/sidekiq/idempotent_jobs.md index a5ae8737ad1..5d1ebce763e 100644 --- a/doc/development/sidekiq/idempotent_jobs.md +++ b/doc/development/sidekiq/idempotent_jobs.md @@ -78,9 +78,8 @@ GitLab supports two deduplication strategies: - `until_executing`, which is the default strategy - `until_executed` -More [deduplication strategies have been -suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/195). If -you are implementing a worker that could benefit from a different +More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/195). +If you are implementing a worker that could benefit from a different strategy, please comment in the issue. #### Until Executing diff --git a/doc/development/sidekiq/index.md b/doc/development/sidekiq/index.md index c9906c4c768..003f54d48b5 100644 --- a/doc/development/sidekiq/index.md +++ b/doc/development/sidekiq/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w We use [Sidekiq](https://github.com/mperham/sidekiq) as our background job processor. These guides are for writing jobs that will work well on GitLab.com and be consistent with our existing worker classes. For -information on administering GitLab, see [configuring Sidekiq](../../administration/sidekiq.md). +information on administering GitLab, see [configuring Sidekiq](../../administration/sidekiq/index.md). There are pages with additional detail on the following topics: @@ -27,12 +27,11 @@ There are pages with additional detail on the following topics: All workers should include `ApplicationWorker` instead of `Sidekiq::Worker`, which adds some convenience methods and automatically sets the queue based on -the [routing rules](../../administration/operations/extra_sidekiq_routing.md#queue-routing-rules). +the [routing rules](../../administration/sidekiq/extra_sidekiq_routing.md#queue-routing-rules). ## Retries -Sidekiq defaults to using [25 -retries](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry), +Sidekiq defaults to using [25 retries](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry), with back-off between each retry. 25 retries means that the last retry would happen around three weeks after the first attempt (assuming all 24 prior retries failed). @@ -64,7 +63,7 @@ error rate. Previously, each worker had its own queue, which was automatically set based on the worker class name. For a worker named `ProcessSomethingWorker`, the queue name would be `process_something`. You can now route workers to a specific queue using -[queue routing rules](../../administration/operations/extra_sidekiq_routing.md#queue-routing-rules). +[queue routing rules](../../administration/sidekiq/extra_sidekiq_routing.md#queue-routing-rules). In GDK, new workers are routed to a queue named `default`. If you're not sure what queue a worker uses, @@ -75,7 +74,7 @@ After adding a new worker, run `bin/rake gitlab:sidekiq:all_queues_yml:generate` to regenerate `app/workers/all_queues.yml` or `ee/app/workers/all_queues.yml` so that it can be picked up by -[`sidekiq-cluster`](../../administration/operations/extra_sidekiq_processes.md) +[`sidekiq-cluster`](../../administration/sidekiq/extra_sidekiq_processes.md) in installations that don't use routing rules. To learn more about potential changes, read [Use routing rules by default and deprecate queue selectors for self-managed](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/596). @@ -176,11 +175,10 @@ available in Sidekiq. There are possible workarounds such as: Some jobs have a weight declared. This is only used when running Sidekiq in the default execution mode - using -[`sidekiq-cluster`](../../administration/operations/extra_sidekiq_processes.md) +[`sidekiq-cluster`](../../administration/sidekiq/extra_sidekiq_processes.md) does not account for weights. -As we are [moving towards using `sidekiq-cluster` in -Free](https://gitlab.com/gitlab-org/gitlab/-/issues/34396), newly-added +As we are [moving towards using `sidekiq-cluster` in Free](https://gitlab.com/gitlab-org/gitlab/-/issues/34396), newly-added workers do not need to have weights specified. They can use the default weight, which is 1. diff --git a/doc/development/sidekiq/logging.md b/doc/development/sidekiq/logging.md index 474ea5de951..b461047ea47 100644 --- a/doc/development/sidekiq/logging.md +++ b/doc/development/sidekiq/logging.md @@ -11,8 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/9) in GitLab 12.8. To have some more information about workers in the logs, we add -[metadata to the jobs in the form of an -`ApplicationContext`](../logging.md#logging-context-metadata-through-rails-or-grape-requests). +[metadata to the jobs in the form of an `ApplicationContext`](../logging.md#logging-context-metadata-through-rails-or-grape-requests). In most cases, when scheduling a job from a request, this context is already deducted from the request and added to the scheduled job. @@ -128,7 +127,7 @@ blocks: ## Arguments logging -As of GitLab 13.6, Sidekiq job arguments are logged by default, unless [`SIDEKIQ_LOG_ARGUMENTS`](../../administration/troubleshooting/sidekiq.md#log-arguments-to-sidekiq-jobs) +As of GitLab 13.6, Sidekiq job arguments are logged by default, unless [`SIDEKIQ_LOG_ARGUMENTS`](../../administration/sidekiq/sidekiq_troubleshooting.md#log-arguments-to-sidekiq-jobs) is disabled. By default, the only arguments logged are numeric arguments, because diff --git a/doc/development/sidekiq/worker_attributes.md b/doc/development/sidekiq/worker_attributes.md index 6820627f761..a1d24d0c392 100644 --- a/doc/development/sidekiq/worker_attributes.md +++ b/doc/development/sidekiq/worker_attributes.md @@ -86,13 +86,11 @@ but that always reduces work. To do this, we want to calculate the expected increase in total execution time and RPS (throughput) for the new shard. We can get these values from: -- The [Queue Detail - dashboard](https://dashboards.gitlab.net/d/sidekiq-queue-detail/sidekiq-queue-detail) +- The [Queue Detail dashboard](https://dashboards.gitlab.net/d/sidekiq-queue-detail/sidekiq-queue-detail) has values for the queue itself. For a new queue, we can look for queues that have similar patterns or are scheduled in similar circumstances. -- The [Shard Detail - dashboard](https://dashboards.gitlab.net/d/sidekiq-shard-detail/sidekiq-shard-detail) +- The [Shard Detail dashboard](https://dashboards.gitlab.net/d/sidekiq-shard-detail/sidekiq-shard-detail) has Total Execution Time and Throughput (RPS). The Shard Utilization panel displays if there is currently any excess capacity for this shard. diff --git a/doc/development/single_table_inheritance.md b/doc/development/single_table_inheritance.md index c8d082e8a67..da8d48f2a42 100644 --- a/doc/development/single_table_inheritance.md +++ b/doc/development/single_table_inheritance.md @@ -1,63 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/single_table_inheritance.md' +remove_date: '2022-11-06' --- -# Single Table Inheritance +This document was moved to [another location](database/single_table_inheritance.md). -**Summary:** don't use Single Table Inheritance (STI), use separate tables -instead. - -Rails makes it possible to have multiple models stored in the same table and map -these rows to the correct models using a `type` column. This can be used to for -example store two different types of SSH keys in the same table. - -While tempting to use one should avoid this at all costs for the same reasons as -outlined in the document ["Polymorphic Associations"](polymorphic_associations.md). - -## Solution - -The solution is very simple: just use a separate table for every type you'd -otherwise store in the same table. For example, instead of having a `keys` table -with `type` set to either `Key` or `DeployKey` you'd have two separate tables: -`keys` and `deploy_keys`. - -## In migrations - -Whenever a model is used in a migration, single table inheritance should be disabled. -Due to the way Rails loads associations (even in migrations), failing to disable STI -could result in loading unexpected code or associations which may cause unintended -side effects or failures during upgrades. - -```ruby -class SomeMigration < Gitlab::Database::Migration[2.0] - class Services < MigrationRecord - self.table_name = 'services' - self.inheritance_column = :_type_disabled - end - - def up - ... -``` - -If nothing needs to be added to the model other than disabling STI or `EachBatch`, -use the helper `define_batchable_model` instead of defining the class. -This ensures that the migration loads the columns for the migration in isolation, -and the helper disables STI by default. - -```ruby -class EnqueueSomeBackgroundMigration < Gitlab::Database::Migration[1.0] - disable_ddl_transaction! - - def up - define_batchable_model('services').select(:id).in_batches do |relation| - jobs = relation.pluck(:id).map do |id| - ['ExtractServicesUrl', [id]] - end - - BackgroundMigrationWorker.bulk_perform_async(jobs) - end - end - ... -``` +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/snowplow/implementation.md b/doc/development/snowplow/implementation.md index f8e37aee1e0..9a923b115a2 100644 --- a/doc/development/snowplow/implementation.md +++ b/doc/development/snowplow/implementation.md @@ -431,7 +431,7 @@ To test backend Snowplow events, use the `expect_snowplow_event` helper. For mor ### Performance -We use the [AsyncEmitter](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/ruby-tracker/emitters/#the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. +We use the [AsyncEmitter](https://snowplow.github.io/snowplow-ruby-tracker/SnowplowTracker/AsyncEmitter.html) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. ## Develop and test Snowplow diff --git a/doc/development/snowplow/index.md b/doc/development/snowplow/index.md index 155ce87b8d9..24cd9093267 100644 --- a/doc/development/snowplow/index.md +++ b/doc/development/snowplow/index.md @@ -89,10 +89,10 @@ Each click event provides attributes that describe the event. | Attribute | Type | Required | Description | | --------- | ------- | -------- | ----------- | -| category | text | true | The page or backend section of the application. Unless infeasible, use the Rails page attribute by default in the frontend, and namespace + class name on the backend. | +| category | text | true | The page or backend section of the application. Unless infeasible, use the Rails page attribute by default in the frontend, and namespace + class name on the backend, for example, `Notes::CreateService`. | | action | text | true | The action the user takes, or aspect that's being instrumented. The first word must describe the action or aspect. For example, clicks must be `click`, activations must be `activate`, creations must be `create`. Use underscores to describe what was acted on. For example, activating a form field is `activate_form_input`, an interface action like clicking on a dropdown is `click_dropdown`, a behavior like creating a project record from the backend is `create_project`. | -| label | text | false | The specific element or object to act on. This can be one of the following: the label of the element, for example, a tab labeled 'Create from template' for `create_from_template`; a unique identifier if no text is available, for example, `groups_dropdown_close` for closing the Groups dropdown in the top bar; or the name or title attribute of a record being created. | -| property | text | false | Any additional property of the element, or object being acted on. | +| label | text | false | The specific element or object to act on. This can be one of the following: the label of the element, for example, a tab labeled 'Create from template' for `create_from_template`; a unique identifier if no text is available, for example, `groups_dropdown_close` for closing the Groups dropdown in the top bar; or the name or title attribute of a record being created. For Service Ping metrics adapted to Snowplow events, this should be the full metric [key path](../service_ping/metrics_dictionary.md#metric-key_path) taken from its definition file. | +| property | text | false | Any additional property of the element, or object being acted on. For Service Ping metrics adapted to Snowplow events, this should be additional information or context that can help analyze the event. For example, in the case of `usage_activity_by_stage_monthly.create.merge_requests_users`, there are four different possible merge request actions: "create", "merge", "comment", and "close". Each of these would be a possible property value. | | value | decimal | false | Describes a numeric value (decimal) directly related to the event. This could be the value of an input. For example, `10` when clicking `internal` visibility. | ### Examples @@ -106,6 +106,7 @@ Each click event provides attributes that describe the event. | `[projects:blob:show]` | `congratulate_first_pipeline` | `click_button` | `[human_access]` | - | | `[projects:clusters:new]` | `chart_options` | `generate_link` | `[chart_link]` | - | | `[projects:clusters:new]` | `chart_options` | `click_add_label_button` | `[label_id]` | - | +| `API::NpmPackages` | `counts.package_events_i_package_push_package_by_deploy_token` | `push_package` | `npm` | - | _* If you choose to omit the category you can use the default._<br> _** Use property for variable strings._ diff --git a/doc/development/snowplow/infrastructure.md b/doc/development/snowplow/infrastructure.md index 758c850e89f..ea4653dc91d 100644 --- a/doc/development/snowplow/infrastructure.md +++ b/doc/development/snowplow/infrastructure.md @@ -50,7 +50,7 @@ See [Snowplow technology 101](https://github.com/snowplow/snowplow/#snowplow-tec ### Pseudonymization -In contrast to a typical Snowplow pipeline, after enrichment, GitLab Snowplow events go through a [pseudonymization service](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization) in the form of an AWS Lambda service before they are stored in S3 storage. +In contrast to a typical Snowplow pipeline, after enrichment, GitLab Snowplow events go through a [pseudonymization service](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/snowplow-pseudonymization) in the form of an AWS Lambda service before they are stored in S3 storage. #### Why events need to be pseudonymized @@ -85,7 +85,7 @@ There are several tools that monitor Snowplow events tracking in different stage - The number of events that successfully reach Snowplow collectors. - The number of events that failed to reach Snowplow collectors. - The number of backend events that were sent. -- [AWS CloudWatch dashboard](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards:name=SnowPlow;start=P3D) monitors the state of the events in a processing pipeline. The pipeline starts from Snowplow collectors, goes through to enrichers and pseudonymization, and then up to persistence in an S3 bucket. From S3, the events are imported into the Snowflake Data Warehouse. You must have AWS access rights to view this dashboard. For more information, see [monitoring](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization#monitoring) in the Snowplow Events pseudonymization service documentation. +- [AWS CloudWatch dashboard](https://console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards:name=SnowPlow;start=P3D) monitors the state of the events in a processing pipeline. The pipeline starts from Snowplow collectors, goes through to enrichers and pseudonymization, and then up to persistence in an S3 bucket. From S3, the events are imported into the Snowflake Data Warehouse. You must have AWS access rights to view this dashboard. For more information, see [monitoring](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/snowplow-pseudonymization#monitoring) in the Snowplow Events pseudonymization service documentation. - [Sisense dashboard](https://app.periscopedata.com/app/gitlab/417669/Snowplow-Summary-Dashboard) provides information about the number of good and bad events imported into the Data Warehouse, in addition to the total number of imported Snowplow events. For more information, see this [video walk-through](https://www.youtube.com/watch?v=NxPS0aKa_oU). @@ -93,7 +93,7 @@ For more information, see this [video walk-through](https://www.youtube.com/watc ## Related topics - [Snowplow technology 101](https://github.com/snowplow/snowplow/#snowplow-technology-101) -- [Snowplow pseudonymization AWS Lambda project](https://gitlab.com/gitlab-org/growth/product-intelligence/snowplow-pseudonymization) +- [Snowplow pseudonymization AWS Lambda project](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/snowplow-pseudonymization) - [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) - [Data Infrastructure](https://about.gitlab.com/handbook/business-technology/data-team/platform/infrastructure/) - [Snowplow architecture overview (internal)](https://www.youtube.com/watch?v=eVYJjzspsLU) diff --git a/doc/development/snowplow/review_guidelines.md b/doc/development/snowplow/review_guidelines.md index 673166452b7..44de849792c 100644 --- a/doc/development/snowplow/review_guidelines.md +++ b/doc/development/snowplow/review_guidelines.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Snowplow review guidelines This page includes introductory material for a -[Product Intelligence](https://about.gitlab.com/handbook/engineering/development/growth/product-intelligence/) +[Product Intelligence](https://about.gitlab.com/handbook/engineering/development/analytics/product-intelligence/) review, and is specific to Snowplow related reviews. For broader advice and general best practices for code reviews, refer to our [code review guide](../code_review.md). diff --git a/doc/development/sql.md b/doc/development/sql.md index 8553e2a5500..7101bf7fb4b 100644 --- a/doc/development/sql.md +++ b/doc/development/sql.md @@ -79,8 +79,9 @@ ON table_name USING GIN(column_name gin_trgm_ops); ``` -The key here is the `GIN(column_name gin_trgm_ops)` part. This creates a [GIN -index](https://www.postgresql.org/docs/current/gin.html) with the operator class set to `gin_trgm_ops`. These indexes +The key here is the `GIN(column_name gin_trgm_ops)` part. This creates a +[GIN index](https://www.postgresql.org/docs/current/gin.html) +with the operator class set to `gin_trgm_ops`. These indexes _can_ be used by `ILIKE` / `LIKE` and can lead to greatly improved performance. One downside of these indexes is that they can easily get quite large (depending on the amount of data indexed). diff --git a/doc/development/stage_group_observability/dashboards/stage_group_dashboard.md b/doc/development/stage_group_observability/dashboards/stage_group_dashboard.md index c1831cfce69..c8c18b93a8f 100644 --- a/doc/development/stage_group_observability/dashboards/stage_group_dashboard.md +++ b/doc/development/stage_group_observability/dashboards/stage_group_dashboard.md @@ -56,7 +56,7 @@ description, note the following: To inspect the raw data of the panel for further calculation, select **Inspect** from the dropdown list of a panel. Queries, raw data, and panel JSON structure are available. -Read more at [Grafana panel inspection](https://grafana.com/docs/grafana/latest/panels/inspect-panel/). +Read more at [Grafana panel inspection](http://grafana.com/docs/grafana/next/panels/query-a-data-source/). All the dashboards are powered by [Grafana](https://grafana.com/), a frontend for displaying metrics. Grafana consumes the data returned from queries to backend Prometheus data source, then presents it diff --git a/doc/development/swapping_tables.md b/doc/development/swapping_tables.md index efb481ccf35..eaa6568dc36 100644 --- a/doc/development/swapping_tables.md +++ b/doc/development/swapping_tables.md @@ -1,51 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/swapping_tables.md' +remove_date: '2022-11-04' --- -# Swapping Tables +This document was moved to [another location](database/swapping_tables.md). -Sometimes you need to replace one table with another. For example, when -migrating data in a very large table it's often better to create a copy of the -table and insert & migrate the data into this new table in the background. - -Let's say you want to swap the table `events` with `events_for_migration`. In -this case you need to follow 3 steps: - -1. Rename `events` to `events_temporary` -1. Rename `events_for_migration` to `events` -1. Rename `events_temporary` to `events_for_migration` - -Rails allows you to do this using the `rename_table` method: - -```ruby -rename_table :events, :events_temporary -rename_table :events_for_migration, :events -rename_table :events_temporary, :events_for_migration -``` - -This does not require any downtime as long as the 3 `rename_table` calls are -executed in the _same_ database transaction. Rails by default uses database -transactions for migrations, but if it doesn't you need to start one -manually: - -```ruby -Event.transaction do - rename_table :events, :events_temporary - rename_table :events_for_migration, :events - rename_table :events_temporary, :events_for_migration -end -``` - -Once swapped you _have to_ reset the primary key of the new table. For -PostgreSQL you can use the `reset_pk_sequence!` method like so: - -```ruby -reset_pk_sequence!('events') -``` - -Failure to reset the primary keys results in newly created rows starting -with an ID value of 1. Depending on the existing data this can then lead to -duplicate key constraints from popping up, preventing users from creating new -data. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index ea36214f6b7..79a72981e3f 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -422,8 +422,8 @@ Use the coverage reports to ensure your tests cover 100% of your code. ### System / Feature tests NOTE: -Before writing a new system test, [please consider **not** -writing one](testing_levels.md#consider-not-writing-a-system-test)! +Before writing a new system test, +[please consider **not** writing one](testing_levels.md#consider-not-writing-a-system-test)! - Feature specs should be named `ROLE_ACTION_spec.rb`, such as `user_changes_password_spec.rb`. @@ -909,8 +909,8 @@ By default, Sidekiq jobs are enqueued into a jobs array and aren't processed. If a test queues Sidekiq jobs and need them to be processed, the `:sidekiq_inline` trait can be used. -The `:sidekiq_might_not_need_inline` trait was added when [Sidekiq inline mode was -changed to fake mode](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/15479) +The `:sidekiq_might_not_need_inline` trait was added when +[Sidekiq inline mode was changed to fake mode](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/15479) to all the tests that needed Sidekiq to actually process jobs. Tests with this trait should be either fixed to not rely on Sidekiq processing jobs, or their `:sidekiq_might_not_need_inline` trait should be updated to `:sidekiq_inline` if @@ -1239,8 +1239,7 @@ The `match_schema` matcher allows validating that the subject matches a a JSON string or a JSON-compatible data structure. `match_response_schema` is a convenience matcher for using with a -response object. from a [request -spec](testing_levels.md#integration-tests). +response object. from a [request spec](testing_levels.md#integration-tests). Examples: diff --git a/doc/development/testing_guide/contract/consumer_tests.md b/doc/development/testing_guide/contract/consumer_tests.md index df7c9ee0abd..46f4f446ad9 100644 --- a/doc/development/testing_guide/contract/consumer_tests.md +++ b/doc/development/testing_guide/contract/consumer_tests.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Writing consumer tests -This tutorial guides you through writing a consumer test from scratch. To start, the consumer tests are written using [`jest-pact`](https://github.com/pact-foundation/jest-pact) that builds on top of [`pact-js`](https://github.com/pact-foundation/pact-js). This tutorial shows you how to write a consumer test for the `/discussions.json` endpoint, which is actually `/:namespace_name/:project_name/-/merge_requests/:id/discussions.json`. +This tutorial guides you through writing a consumer test from scratch. To start, the consumer tests are written using [`jest-pact`](https://github.com/pact-foundation/jest-pact) that builds on top of [`pact-js`](https://github.com/pact-foundation/pact-js). This tutorial shows you how to write a consumer test for the `/discussions.json` REST API endpoint, which is actually `/:namespace_name/:project_name/-/merge_requests/:id/discussions.json`. For an example of a GraphQL consumer test, see [`spec/contracts/consumer/specs/project/pipeline/show.spec.js`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/spec/contracts/consumer/specs/project/pipeline/show.spec.js). ## Create the skeleton @@ -24,7 +24,7 @@ To learn more about how the contract test directory is structured, see the contr The Pact consumer test is defined through the `pactWith` function that takes `PactOptions` and the `PactFn`. ```javascript -const { pactWith } = require('jest-pact'); +import { pactWith } from 'jest-pact'; pactWith(PactOptions, PactFn); ``` @@ -34,7 +34,7 @@ pactWith(PactOptions, PactFn); `PactOptions` with `jest-pact` introduces [additional options](https://github.com/pact-foundation/jest-pact/blob/dce370c1ab4b7cb5dff12c4b62246dc229c53d0e/README.md#defaults) that build on top of the ones [provided in `pact-js`](https://github.com/pact-foundation/pact-js#constructor). In most cases, you define the `consumer`, `provider`, `log`, and `dir` options for these tests. ```javascript -const { pactWith } = require('jest-pact'); +import { pactWith } from 'jest-pact'; pactWith( { @@ -54,7 +54,7 @@ To learn more about how to name the consumers and providers, see contract testin The `PactFn` is where your tests are defined. This is where you set up the mock provider and where you can use the standard Jest methods like [`Jest.describe`](https://jestjs.io/docs/api#describename-fn), [`Jest.beforeEach`](https://jestjs.io/docs/api#beforeeachfn-timeout), and [`Jest.it`](https://jestjs.io/docs/api#testname-fn-timeout). For more information, see [https://jestjs.io/docs/api](https://jestjs.io/docs/api). ```javascript -const { pactWith } = require('jest-pact'); +import { pactWith } from 'jest-pact'; pactWith( { @@ -70,7 +70,7 @@ pactWith( }); - it('return a successful body', () => { + it('return a successful body', async () => { }); }); @@ -92,8 +92,8 @@ For this tutorial, define four attributes for the `Interaction`: After you define the `Interaction`, add that interaction to the mock provider by calling `addInteraction`. ```javascript -const { pactWith } = require('jest-pact'); -const { Matchers } = require('@pact-foundation/pact'); +import { pactWith } from 'jest-pact'; +import { Matchers } from '@pact-foundation/pact'; pactWith( { @@ -132,7 +132,7 @@ pactWith( provider.addInteraction(interaction); }); - it('return a successful body', () => { + it('return a successful body', async () => { }); }); @@ -142,38 +142,36 @@ pactWith( ### Response body `Matchers` -Notice how we use `Matchers` in the `body` of the expected response. This allows us to be flexible enough to accept different values but still be strict enough to distinguish between valid and invalid values. We must ensure that we have a tight definition that is neither too strict nor too lax. Read more about the [different types of `Matchers`](https://github.com/pact-foundation/pact-js#using-the-v3-matching-rules). +Notice how we use `Matchers` in the `body` of the expected response. This allows us to be flexible enough to accept different values but still be strict enough to distinguish between valid and invalid values. We must ensure that we have a tight definition that is neither too strict nor too lax. Read more about the [different types of `Matchers`](https://github.com/pact-foundation/pact-js/blob/master/docs/matching.md). We are currently using the V2 matching rules. ## Write the test After the mock provider is set up, you can write the test. For this test, you make a request and expect a particular response. -First, set up the client that makes the API request. To do that, create `spec/contracts/consumer/endpoints/project/merge_requests.js` and add the following API request. +First, set up the client that makes the API request. To do that, create `spec/contracts/consumer/resources/api/project/merge_requests.js` and add the following API request. If the endpoint is a GraphQL, then we create it under `spec/contracts/consumer/resources/graphql` instead. ```javascript -const axios = require('axios'); - -exports.getDiscussions = (endpoint) => { - const url = endpoint.url; - - return axios - .request({ - method: 'GET', - baseURL: url, - url: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', - headers: { Accept: '*/*' }, - }) - .then((response) => response.data); -}; +import axios from 'axios'; + +export async function getDiscussions(endpoint) { + const { url } = endpoint; + + return axios({ + method: 'GET', + baseURL: url, + url: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', + headers: { Accept: '*/*' }, + }) +} ``` After that's set up, import it to the test file and call it to make the request. Then, you can make the request and define your expectations. ```javascript -const { pactWith } = require('jest-pact'); -const { Matchers } = require('@pact-foundation/pact'); +import { pactWith } from 'jest-pact'; +import { Matchers } from '@pact-foundation/pact'; -const { getDiscussions } = require('../endpoints/project/merge_requests'); +import { getDiscussions } from '../../../resources/api/project/merge_requests'; pactWith( { @@ -211,17 +209,17 @@ pactWith( }; }); - it('return a successful body', () => { - return getDiscussions({ + it('return a successful body', async () => { + const discussions = await getDiscussions({ url: provider.mockService.baseUrl, - }).then((discussions) => { - expect(discussions).toEqual(Matchers.eachLike({ - id: 'fd73763cbcbf7b29eb8765d969a38f7d735e222a', - project_id: 6954442, - ... - resolved: true - })); }); + + expect(discussions).toEqual(Matchers.eachLike({ + id: 'fd73763cbcbf7b29eb8765d969a38f7d735e222a', + project_id: 6954442, + ... + resolved: true + })); }); }); }, @@ -237,7 +235,7 @@ As you may have noticed, the request and response definitions can get large. Thi Create a file under `spec/contracts/consumer/fixtures/project/merge_request` called `discussions.fixture.js` where you will place the `request` and `response` definitions. ```javascript -const { Matchers } = require('@pact-foundation/pact'); +import { Matchers } from '@pact-foundation/pact'; const body = Matchers.eachLike({ id: Matchers.string('fd73763cbcbf7b29eb8765d969a38f7d735e222a'), @@ -254,11 +252,15 @@ const Discussions = { headers: { 'Content-Type': 'application/json; charset=utf-8', }, - body: body, + body, }, - request: { + scenario: { + state: 'a merge request with discussions exists', uponReceiving: 'a request for discussions', + }, + + request: { withRequest: { method: 'GET', path: '/gitlab-org/gitlab-qa/-/merge_requests/1/discussions.json', @@ -275,36 +277,41 @@ exports.Discussions = Discussions; With all of that moved to the `fixture`, you can simplify the test to the following: ```javascript -const { pactWith } = require('jest-pact'); +import { pactWith } from 'jest-pact'; + +import { Discussions } from '../../../fixtures/project/merge_request/discussions.fixture'; +import { getDiscussions } from '../../../resources/api/project/merge_requests'; -const { Discussions } = require('../fixtures/discussions.fixture'); -const { getDiscussions } = require('../endpoints/project/merge_requests'); +const CONSUMER_NAME = 'MergeRequest#show'; +const PROVIDER_NAME = 'Merge Request Discussions Endpoint'; +const CONSUMER_LOG = '../logs/consumer.log'; +const CONTRACT_DIR = '../contracts/project/merge_request/show'; pactWith( { - consumer: 'MergeRequest#show', - provider: 'Merge Request Discussions Endpoint', - log: '../logs/consumer.log', - dir: '../contracts/project/merge_request/show', + consumer: CONSUMER_NAME, + provider: PROVIDER_NAME, + log: CONSUMER_LOG, + dir: CONTRACT_DIR, }, (provider) => { - describe('Merge Request Discussions Endpoint', () => { + describe(PROVIDER_NAME, () => { beforeEach(() => { const interaction = { - state: 'a merge request with discussions exists', + ...Discussions.scenario, ...Discussions.request, willRespondWith: Discussions.success, }; - return provider.addInteraction(interaction); + provider.addInteraction(interaction); }); - it('return a successful body', () => { - return getDiscussions({ + it('return a successful body', async () => { + const discussions = await getDiscussions({ url: provider.mockService.baseUrl, - }).then((discussions) => { - expect(discussions).toEqual(Discussions.body); }); + + expect(discussions).toEqual(Discussions.body); }); }); }, diff --git a/doc/development/testing_guide/contract/index.md b/doc/development/testing_guide/contract/index.md index 8e12eea2874..30a4adaca44 100644 --- a/doc/development/testing_guide/contract/index.md +++ b/doc/development/testing_guide/contract/index.md @@ -28,14 +28,14 @@ Before running the consumer tests, go to `spec/contracts/consumer` and run `npm ### Run the provider tests -Before running the provider tests, make sure your GDK (GitLab Development Kit) is fully set up and running. You can follow the setup instructions detailed in the [GDK repository](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main). To run the provider tests, you use Rake tasks that are defined in [`./lib/tasks/contracts.rake`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/tasks/contracts.rake). To get a list of all the Rake tasks related to the provider tests, run `bundle exec rake -T contracts`. For example: +Before running the provider tests, make sure your GDK (GitLab Development Kit) is fully set up and running. You can follow the setup instructions detailed in the [GDK repository](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main). To run the provider tests, you use Rake tasks that can be found in [`./lib/tasks/contracts`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/tasks/contracts). To get a list of all the Rake tasks related to the provider tests, run `bundle exec rake -T contracts`. For example: ```shell $ bundle exec rake -T contracts -rake contracts:mr:pact:verify:diffs # Verify provider against the consumer pacts for diffs -rake contracts:mr:pact:verify:discussions # Verify provider against the consumer pacts for discussions -rake contracts:mr:pact:verify:metadata # Verify provider against the consumer pacts for metadata -rake contracts:mr:test:merge_request[contract_mr] # Run all merge request contract tests +rake contracts:merge_requests:pact:verify:diffs_batch # Verify provider against the consumer pacts for diffs_batch +rake contracts:merge_requests:pact:verify:diffs_metadata # Verify provider against the consumer pacts for diffs_metadata +rake contracts:merge_requests:pact:verify:discussions # Verify provider against the consumer pacts for discussions +rake contracts:merge_requests:test:merge_requests[contract_merge_requests] # Run all merge request contract tests ``` ## Test suite folder structure and naming conventions @@ -50,11 +50,11 @@ Having an organized and sensible folder structure for the test suite makes it ea The consumer tests are grouped according to the different pages in the application. Each file contains various types of requests found in a page. As such, the consumer test files are named using the Rails standards of how pages are referenced. For example, the project pipelines page would be the `Project::Pipeline#index` page so the equivalent consumer test would be located in `consumer/specs/project/pipelines/index.spec.js`. -When defining the location to output the contract generated by the test, we want to follow the same file structure which would be `contracts/project/pipelines/` for this example. This is the structure in `consumer/endpoints` and `consumer/fixtures` as well. +When defining the location to output the contract generated by the test, we want to follow the same file structure which would be `contracts/project/pipelines/` for this example. This is the structure in `consumer/resources` and `consumer/fixtures` as well. #### Provider tests -The provider tests are grouped similarly to our controllers. Each of these tests contains various tests for an API endpoint. For example, the API endpoint to get a list of pipelines for a project would be located in `provider/pact_helpers/project/pipelines/get_list_project_pipelines_helper.rb`. The provider states are structured the same way. +The provider tests are grouped similarly to our controllers. Each of these tests contains various tests for an API endpoint. For example, the API endpoint to get a list of pipelines for a project would be located in `provider/pact_helpers/project/pipelines/get_list_project_pipelines_helper.rb`. The provider states are grouped according to the different pages in the application similar to the consumer tests. ### Naming conventions diff --git a/doc/development/testing_guide/end_to_end/best_practices.md b/doc/development/testing_guide/end_to_end/best_practices.md index 00b843ffdbe..bfda94b1f1d 100644 --- a/doc/development/testing_guide/end_to_end/best_practices.md +++ b/doc/development/testing_guide/end_to_end/best_practices.md @@ -415,8 +415,8 @@ except(page).to have_no_text('hidden') Unfortunately, that's not automatically the case for the predicate methods that we add to our [page objects](page_objects.md). We need to [create our own negatable matchers](https://relishapp.com/rspec/rspec-expectations/v/3-9/docs/custom-matchers/define-a-custom-matcher#matcher-with-separate-logic-for-expect().to-and-expect().not-to). -The initial example uses the `have_job` matcher which is derived from the [`has_job?` predicate -method of the `Page::Project::Pipeline::Show` page object](https://gitlab.com/gitlab-org/gitlab/-/blob/87864b3047c23b4308f59c27a3757045944af447/qa/qa/page/project/pipeline/show.rb#L53). +The initial example uses the `have_job` matcher which is derived from the +[`has_job?` predicate method of the `Page::Project::Pipeline::Show` page object](https://gitlab.com/gitlab-org/gitlab/-/blob/87864b3047c23b4308f59c27a3757045944af447/qa/qa/page/project/pipeline/show.rb#L53). To create a negatable matcher, we use `has_no_job?` for the negative case: ```ruby diff --git a/doc/development/testing_guide/end_to_end/feature_flags.md b/doc/development/testing_guide/end_to_end/feature_flags.md index cb4c8e8a6e8..33f73304a26 100644 --- a/doc/development/testing_guide/end_to_end/feature_flags.md +++ b/doc/development/testing_guide/end_to_end/feature_flags.md @@ -217,8 +217,8 @@ If enabling the feature flag results in E2E test failures, you can browse the ar If an end-to-end test enables a feature flag, the end-to-end test suite can be used to test changes in a merge request by running the `package-and-qa` job in the merge request pipeline. If the feature flag and relevant changes have already been merged, you can confirm that the tests -pass on the default branch. The end-to-end tests run on the default branch every two hours, and the results are posted to a [Test -Session Report, which is available in the testcase-sessions project](https://gitlab.com/gitlab-org/quality/testcase-sessions/-/issues?label_name%5B%5D=found%3Amain). +pass on the default branch. The end-to-end tests run on the default branch every two hours, and the results are posted to a +[Test Session Report, which is available in the testcase-sessions project](https://gitlab.com/gitlab-org/quality/testcase-sessions/-/issues?label_name%5B%5D=found%3Amain). If the relevant tests do not enable the feature flag themselves, you can check if the tests will need to be updated by opening a draft merge request that enables the flag by default via a [feature flag definition file](../../feature_flags/index.md#feature-flag-definition-and-validation). diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 06359d612ad..989d090d581 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -140,8 +140,8 @@ a flaky test we first want to make sure that it's no longer flaky. We can do that using the `ce:custom-parallel` and `ee:custom-parallel` jobs. Both are manual jobs that you can configure using custom variables. When clicking the name (not the play icon) of one of the parallel jobs, -you are prompted to enter variables. You can use any of [the variables -that can be used with `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#supported-gitlab-environment-variables) +you are prompted to enter variables. You can use any of +[the variables that can be used with `gitlab-qa`](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#supported-gitlab-environment-variables) as well as these: | Variable | Description | @@ -150,8 +150,9 @@ as well as these: | `QA_TESTS` | The tests to run (no default, which means run all the tests in the scenario). Use file paths as you would when running tests via RSpec, for example, `qa/specs/features/ee/browser_ui` would include all the `EE` UI tests. | | `QA_RSPEC_TAGS` | The RSpec tags to add (no default) | -For now, [manual jobs with custom variables don't use the same variable -when retried](https://gitlab.com/gitlab-org/gitlab/-/issues/31367), so if you want to run the same tests multiple times, +For now, +[manual jobs with custom variables don't use the same variable when retried](https://gitlab.com/gitlab-org/gitlab/-/issues/31367), +so if you want to run the same tests multiple times, specify the same variables in each `custom-parallel` job (up to as many of the 10 available jobs that you want to run). @@ -164,8 +165,8 @@ automatically started: it runs the QA smoke suite against the You can also manually start the `review-qa-all`: it runs the full QA suite against the [Review App](../review_apps.md). -**This runs end-to-end tests against a Review App based on [the official GitLab -Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/), itself deployed with custom +**This runs end-to-end tests against a Review App based on +[the official GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/), itself deployed with custom [Cloud Native components](https://gitlab.com/gitlab-org/build/CNG) built from your merge request's changes.** See [Review Apps](../review_apps.md) for more details about Review Apps. @@ -235,7 +236,7 @@ Each type of scheduled pipeline generates a static link for the latest test repo - [`staging-sanity`](https://storage.googleapis.com/gitlab-qa-allure-reports/staging-sanity/master/index.html) - [`staging-sanity-no-admin`](https://storage.googleapis.com/gitlab-qa-allure-reports/staging-sanity-no-admin/master/index.html) - [`canary-sanity`](https://storage.googleapis.com/gitlab-qa-allure-reports/canary-sanity/master/index.html) -- [`production`](https://storage.googleapis.com/gitlab-qa-allure-reports/production/master/index.html) +- [`production`](https://storage.googleapis.com/gitlab-qa-allure-reports/production-full/master/index.html) - [`production-sanity`](https://storage.googleapis.com/gitlab-qa-allure-reports/production-sanity/master/index.html) ## How do I run the tests? @@ -243,8 +244,8 @@ Each type of scheduled pipeline generates a static link for the latest test repo If you are not [testing code in a merge request](#testing-code-in-merge-requests), there are two main options for running the tests. If you want to run the existing tests against a live GitLab instance or against a pre-built Docker image, -use the [GitLab QA orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md). See also [examples -of the test scenarios you can run via the orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#examples). +use the [GitLab QA orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md). See also +[examples of the test scenarios you can run via the orchestrator](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/docs/what_tests_can_be_run.md#examples). On the other hand, if you would like to run against a local development GitLab environment, you can use the [GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit/). @@ -262,8 +263,8 @@ architecture. See the [documentation about it](https://gitlab.com/gitlab-org/git Once you decided where to put [test environment orchestration scenarios](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/lib/gitlab/qa/scenario) and [instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features), take a look at the [GitLab QA README](https://gitlab.com/gitlab-org/gitlab/-/tree/master/qa/README.md), -the [GitLab QA orchestrator README](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md), and [the already existing -instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features). +the [GitLab QA orchestrator README](https://gitlab.com/gitlab-org/gitlab-qa/tree/master/README.md), +and [the already existing instance-level scenarios](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa/qa/specs/features). ### Consider **not** writing an end-to-end test diff --git a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md index 591d03db7b8..322f2412e5b 100644 --- a/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md +++ b/doc/development/testing_guide/end_to_end/rspec_metadata_tests.md @@ -30,6 +30,7 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec | `:ldap_no_tls` | The test requires a GitLab instance to be configured to use an external LDAP server with TLS not enabled. | | `:ldap_tls` | The test requires a GitLab instance to be configured to use an external LDAP server with TLS enabled. | | `:mattermost` | The test requires a GitLab Mattermost service on the GitLab instance. | +| `:metrics` | The test requires a GitLab instance where [dedicated metrics exporters](../../../administration/monitoring/prometheus/web_exporter.md) are running alongside Puma and Sidekiq. | | `:mixed_env` | The test should only be executed in environments that have a paired canary version available through traffic routing based on the existence of the `gitlab_canary=true` cookie. Tests in this category are switching the cookie mid-test to validate mixed deployment environments. | | `:object_storage` | The test requires a GitLab instance to be configured to use multiple [object storage types](../../../administration/object_storage.md). Uses MinIO as the object storage server. | | `:only` | The test is only to be run in specific execution contexts. See [test execution context selection](execution_context_selection.md) for more information. | diff --git a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md index 438294161ac..322f108783f 100644 --- a/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md +++ b/doc/development/testing_guide/end_to_end/running_tests_that_require_special_setup.md @@ -8,11 +8,20 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Jenkins spec -The [`jenkins_build_status_spec`](https://gitlab.com/gitlab-org/gitlab/-/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) spins up a Jenkins instance in a Docker container based on an image stored in the [GitLab-QA container registry](https://gitlab.com/gitlab-org/gitlab-qa/container_registry). -The Docker image it uses is preconfigured with some base data and plugins. -The test then configures the GitLab plugin in Jenkins with a URL of the GitLab instance that are used -to run the tests. Unfortunately, the GitLab Jenkins plugin does not accept ports so `http://localhost:3000` would -not be accepted. Therefore, this requires us to run GitLab on port 80 or inside a Docker container. +The [`jenkins_build_status_spec`](https://gitlab.com/gitlab-org/gitlab/-/blob/24a86debf49f3aed6f2ecfd6e8f9233b3a214181/qa/qa/specs/features/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb) +spins up a Jenkins instance in a Docker container with the Jenkins GitLab plugin pre-installed. Due to a license restriction we are unable to distribute this image. +To build a QA compatible image, please visit the [third party images project](https://gitlab.com/gitlab-org/quality/third-party-docker-public), where third party Dockerfiles can be found. +The project also has instructions for forking and building the images automatically in CI. + +Some extra environment variables for the location of the forked repository are also needed. + +- `QA_THIRD_PARTY_DOCKER_REGISTRY` (the container registry where the repository/images are hosted, eg `registry.gitlab.com`) +- `QA_THIRD_PARTY_DOCKER_REPOSITORY` (the base repository path where the images are hosted, eg `registry.gitlab.com/<project path>`) +- `QA_THIRD_PARTY_DOCKER_USER` (a username that has access to the container registry for this repository) +- `QA_THIRD_PARTY_DOCKER_PASSWORD` (a password/token for the username to authenticate with) + +The test configures the GitLab plugin in Jenkins with a URL of the GitLab instance that are used +to run the tests. Bi-directional networking is needed between a GitLab instance and Jenkins, so GitLab can also be started in a Docker container. To start a Docker container for GitLab based on the nightly image: @@ -21,34 +30,25 @@ docker run \ --publish 80:80 \ --name gitlab \ --hostname localhost \ + --network test gitlab/gitlab-ee:nightly ``` To run the tests from the `/qa` directory: ```shell -WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All http://localhost -- qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb +export QA_THIRD_PARTY_DOCKER_REGISTRY=<registry> +export QA_THIRD_PARTY_DOCKER_REPOSITORY=<repository> +export QA_THIRD_PARTY_DOCKER_USER=<user with registry access> +export QA_THIRD_PARTY_DOCKER_PASSWORD=<password for user> +export WEBDRIVER_HEADLESS=0 +bin/qa Test::Instance::All http://localhost -- qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb ``` The test automatically spins up a Docker container for Jenkins and tear down once the test completes. -However, if you need to run Jenkins manually outside of the tests, use this command: - -```shell -docker run \ - --hostname localhost \ - --name jenkins-server \ - --env JENKINS_HOME=jenkins_home \ - --publish 8080:8080 \ - registry.gitlab.com/gitlab-org/gitlab-qa/jenkins-gitlab:version1 -``` - -Jenkins is available on `http://localhost:8080`. - -Administrator username is `admin` and password is `password`. - -It is worth noting that this is not an orchestrated test. It is [tagged with the `:orchestrated` meta](https://gitlab.com/gitlab-org/gitlab/-/blob/163c8a8c814db26d11e104d1cb2dcf02eb567dbe/qa/qa/specs/features/ee/browser_ui/3_create/jenkins/jenkins_build_status_spec.rb#L5) -only to prevent it from running in the pipelines for live environments such as Staging. +If you need to run Jenkins manually outside of the tests, please refer to the README for the +[third party images project](https://gitlab.com/gitlab-org/quality/third-party-docker-public/-/blob/main/jenkins/README.md) ### Troubleshooting @@ -385,6 +385,43 @@ To run the LDAP tests on your local with TLS disabled, follow these steps: GITLAB_LDAP_USERNAME="tanuki" GITLAB_LDAP_PASSWORD="password" QA_LOG_LEVEL=debug WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All http://localhost qa/specs/features/browser_ui/1_manage/login/log_into_gitlab_via_ldap_spec.rb ``` +## SMTP tests + +Tests that are tagged with `:smtp` meta tag are orchestrated tests that ensure email notifications are received by a user. + +These tests require a GitLab instance with SMTP enabled and integrated with an SMTP server, [MailHog](https://github.com/mailhog/MailHog). + +To run these tests locally against the GDK: + +1. Add these settings to your `gitlab.yml` file: + + ```yaml + smtp: + enabled: true + address: "mailhog.test" + port: 1025 + ``` + +1. Start MailHog in a Docker container: + + ```shell + docker network create test && docker run \ + --network test \ + --hostname mailhog.test \ + --name mailhog \ + --publish 1025:1025 \ + --publish 8025:8025 \ + mailhog/mailhog:v1.0.0 + ``` + +1. Run the test from [`gitlab/qa`](https://gitlab.com/gitlab-org/gitlab/-/tree/d5447ebb5f99d4c72780681ddf4dc25b0738acba/qa) directory: + + ```shell + QA_LOG_LEVEL=debug WEBDRIVER_HEADLESS=false bin/qa Test::Instance::All http://localhost:3000 qa/specs/features/browser_ui/2_plan/email/trigger_email_notification_spec.rb -- --tag orchestrated + ``` + +For instructions on how to run these tests using the `gitlab-qa` gem, please refer to [the GitLab QA documentation](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/docs/what_tests_can_be_run.md#testintegrationsmtp-ceeefull-image-address). + ## Guide to the mobile suite ### What are mobile tests diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index d91c53823e2..2845dde9a24 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -51,15 +51,12 @@ The default timeout for Jest is set in If your test exceeds that time, it fails. If you cannot improve the performance of the tests, you can increase the timeout -for a specific test using -[`setTestTimeout`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/frontend/__helpers__/timeout.js). +for a specific test using [`jest.setTimeout`](https://jestjs.io/docs/27.x/jest-object#jestsettimeouttimeout) ```javascript -import { setTestTimeout } from 'helpers/timeout'; - describe('Component', () => { it('does something amazing', () => { - setTestTimeout(500); + jest.setTimeout(500); // ... }); }); @@ -466,7 +463,7 @@ it('waits for an Ajax call', () => { #### Vue rendering -Use [`nextTick()`](https://vuejs.org/v2/api/#Vue-nextTick) to wait until a Vue component is +Use [`nextTick()`](https://v2.vuejs.org/v2/api/#Vue-nextTick) to wait until a Vue component is re-rendered. **in Jest:** diff --git a/doc/development/testing_guide/index.md b/doc/development/testing_guide/index.md index fa9f1f1ac3e..cd7c70e2eaa 100644 --- a/doc/development/testing_guide/index.md +++ b/doc/development/testing_guide/index.md @@ -9,8 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w This document describes various guidelines and best practices for automated testing of the GitLab project. -It is meant to be an _extension_ of the [Thoughtbot testing -style guide](https://github.com/thoughtbot/guides/tree/master/testing-rspec). If +It is meant to be an _extension_ of the +[Thoughtbot testing style guide](https://github.com/thoughtbot/guides/tree/master/testing-rspec). If this guide defines a rule that contradicts the Thoughtbot guide, this guide takes precedence. Some guidelines may be repeated verbatim to stress their importance. diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index 532bb9fcdef..b272d23522e 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -25,7 +25,7 @@ For any of the following scenarios, the `start-review-app-pipeline` job would be On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) in the `qa` stage (which comes after the `review` stage), the `review-qa-smoke` and `review-qa-reliable` jobs are automatically started. The `review-qa-smoke` runs -the QA smoke suite and the `review-qa-reliable` executes E2E tests identified as [reliable](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/reliable-tests). +the QA smoke suite and the `review-qa-reliable` executes E2E tests identified as [reliable](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/reliable-tests/). `review-qa-*` jobs ensure that end-to-end tests for the changes in the merge request pass in a live environment. This shifts the identification of e2e failures from an environment on the path to production to the merge request, to prevent breaking features on GitLab.com or costly GitLab.com deployment blockers. `review-qa-*` failures should be investigated with counterpart SET involvement if needed to help determine the root cause of the error. diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index 02f32a031dc..c1bf3609b53 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -115,7 +115,7 @@ graph RL Testing the value of a constant means copying it, resulting in extra effort without additional confidence that the value is correct. - **Vue components**: Computed properties, methods, and lifecycle hooks can be considered an implementation detail of components, are implicitly covered by component tests, and don't need to be tested. - For more information, see the [official Vue guidelines](https://vue-test-utils.vuejs.org/guides/#getting-started). + For more information, see the [official Vue guidelines](https://v1.test-utils.vuejs.org/guides/#getting-started). #### What to mock in unit tests @@ -208,7 +208,7 @@ graph RL Similar to unit tests, background operations cannot be stopped or waited on. This means they continue running in the following tests and cause side effects. - **Child components**: Every component is tested individually, so child components are mocked. - See also [`shallowMount()`](https://vue-test-utils.vuejs.org/api/#shallowmount) + See also [`shallowMount()`](https://v1.test-utils.vuejs.org/api/#shallowmount) #### What *not* to mock in component tests diff --git a/doc/development/testing_guide/testing_migrations_guide.md b/doc/development/testing_guide/testing_migrations_guide.md index d71788e21f3..261a4f4a27e 100644 --- a/doc/development/testing_guide/testing_migrations_guide.md +++ b/doc/development/testing_guide/testing_migrations_guide.md @@ -317,8 +317,8 @@ To test these you usually have to: - Verify that the expected jobs were scheduled, with the correct set of records, the correct batch size, interval, etc. -The behavior of the background migration itself needs to be verified in a [separate -test for the background migration class](#example-background-migration-test). +The behavior of the background migration itself needs to be verified in a +[separate test for the background migration class](#example-background-migration-test). This spec tests the [`db/post_migrate/20210701111909_backfill_issues_upvotes_count.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/v14.1.0-ee/db/post_migrate/20210701111909_backfill_issues_upvotes_count.rb) diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index 17fcd5b3e88..72c3df11a96 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -1,829 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/understanding_explain_plans.md' +remove_date: '2022-11-04' --- -# Understanding EXPLAIN plans +This document was moved to [another location](database/understanding_explain_plans.md). -PostgreSQL allows you to obtain query plans using the `EXPLAIN` command. This -command can be invaluable when trying to determine how a query performs. -You can use this command directly in your SQL query, as long as the query starts -with it: - -```sql -EXPLAIN -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -When running this on GitLab.com, we are presented with the following output: - -```sql -Aggregate (cost=922411.76..922411.77 rows=1 width=8) - -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) -``` - -When using _just_ `EXPLAIN`, PostgreSQL does not actually execute our query, -instead it produces an _estimated_ execution plan based on the available -statistics. This means the actual plan can differ quite a bit. Fortunately, -PostgreSQL provides us with the option to execute the query as well. To do so, -we need to use `EXPLAIN ANALYZE` instead of just `EXPLAIN`: - -```sql -EXPLAIN ANALYZE -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -This produces: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -As we can see this plan is quite different, and includes a lot more data. Let's -discuss this step by step. - -Because `EXPLAIN ANALYZE` executes the query, care should be taken when using a -query that writes data or might time out. If the query modifies data, -consider wrapping it in a transaction that rolls back automatically like so: - -```sql -BEGIN; -EXPLAIN ANALYZE -DELETE FROM users WHERE id = 1; -ROLLBACK; -``` - -The `EXPLAIN` command also takes additional options, such as `BUFFERS`: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -This then produces: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - Buffers: shared hit=208846 - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -For more information, refer to the official -[`EXPLAIN` documentation](https://www.postgresql.org/docs/current/sql-explain.html) -and [using `EXPLAIN` guide](https://www.postgresql.org/docs/current/using-explain.html). - -## Nodes - -Every query plan consists of nodes. Nodes can be nested, and are executed from -the inside out. This means that the innermost node is executed before an outer -node. This can be best thought of as nested function calls, returning their -results as they unwind. For example, a plan starting with an `Aggregate` -followed by a `Nested Loop`, followed by an `Index Only scan` can be thought of -as the following Ruby code: - -```ruby -aggregate( - nested_loop( - index_only_scan() - index_only_scan() - ) -) -``` - -Nodes are indicated using a `->` followed by the type of node taken. For -example: - -```sql -Aggregate (cost=922411.76..922411.77 rows=1 width=8) - -> Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) -``` - -Here the first node executed is `Seq scan on projects`. The `Filter:` is an -additional filter applied to the results of the node. A filter is very similar -to Ruby's `Array#select`: it takes the input rows, applies the filter, and -produces a new list of rows. After the node is done, we perform the `Aggregate` -above it. - -Nested nodes look like this: - -```sql -Aggregate (cost=176.97..176.98 rows=1 width=8) (actual time=0.252..0.252 rows=1 loops=1) - Buffers: shared hit=155 - -> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) - Buffers: shared hit=155 - -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) - Index Cond: (id < 100) - Heap Fetches: 0 - -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) - Index Cond: (id = users_1.id) - Heap Fetches: 0 -Planning time: 2.585 ms -Execution time: 0.310 ms -``` - -Here we first perform two separate "Index Only" scans, followed by performing a -"Nested Loop" on the result of these two scans. - -## Node statistics - -Each node in a plan has a set of associated statistics, such as the cost, the -number of rows produced, the number of loops performed, and more. For example: - -```sql -Seq Scan on projects (cost=0.00..908044.47 rows=5746914 width=0) -``` - -Here we can see that our cost ranges from `0.00..908044.47` (we cover this in -a moment), and we estimate (since we're using `EXPLAIN` and not `EXPLAIN -ANALYZE`) a total of 5,746,914 rows to be produced by this node. The `width` -statistics describes the estimated width of each row, in bytes. - -The `costs` field specifies how expensive a node was. The cost is measured in -arbitrary units determined by the query planner's cost parameters. What -influences the costs depends on a variety of settings, such as `seq_page_cost`, -`cpu_tuple_cost`, and various others. -The format of the costs field is as follows: - -```sql -STARTUP COST..TOTAL COST -``` - -The startup cost states how expensive it was to start the node, with the total -cost describing how expensive the entire node was. In general: the greater the -values, the more expensive the node. - -When using `EXPLAIN ANALYZE`, these statistics also include the actual time -(in milliseconds) spent, and other runtime statistics (for example, the actual number of -produced rows): - -```sql -Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) -``` - -Here we can see we estimated 5,746,969 rows to be returned, but in reality we -returned 5,746,940 rows. We can also see that _just_ this sequential scan took -2.98 seconds to run. - -Using `EXPLAIN (ANALYZE, BUFFERS)` also gives us information about the -number of rows removed by a filter, the number of buffers used, and more. For -example: - -```sql -Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -``` - -Here we can see that our filter has to remove 65,677 rows, and that we use -208,846 buffers. Each buffer in PostgreSQL is 8 KB (8192 bytes), meaning our -above node uses *1.6 GB of buffers*. That's a lot! - -Keep in mind that some statistics are per-loop averages, while others are total values: - -| Field name | Value type | -| --- | --- | -| Actual Total Time | per-loop average | -| Actual Rows | per-loop average | -| Buffers Shared Hit | total value | -| Buffers Shared Read | total value | -| Buffers Shared Dirtied | total value | -| Buffers Shared Written | total value | -| I/O Read Time | total value | -| I/O Read Write | total value | - -For example: - -```sql - -> Index Scan using users_pkey on public.users (cost=0.43..3.44 rows=1 width=1318) (actual time=0.025..0.025 rows=1 loops=888) - Index Cond: (users.id = issues.author_id) - Buffers: shared hit=3543 read=9 - I/O Timings: read=17.760 write=0.000 -``` - -Here we can see that this node used 3552 buffers (3543 + 9), returned 888 rows (`888 * 1`), and the actual duration was 22.2 milliseconds (`888 * 0.025`). -17.76 milliseconds of the total duration was spent in reading from disk, to retrieve data that was not in the cache. - -## Node types - -There are quite a few different types of nodes, so we only cover some of the -more common ones here. - -A full list of all the available nodes and their descriptions can be found in -the [PostgreSQL source file `plannodes.h`](https://gitlab.com/postgres/postgres/blob/master/src/include/nodes/plannodes.h). -pgMustard's [EXPLAIN docs](https://www.pgmustard.com/docs/explain) also offer detailed look into nodes and their fields. - -### Seq Scan - -A sequential scan over (a chunk of) a database table. This is like using -`Array#each`, but on a database table. Sequential scans can be quite slow when -retrieving lots of rows, so it's best to avoid these for large tables. - -### Index Only Scan - -A scan on an index that did not require fetching anything from the table. In -certain cases an index only scan may still fetch data from the table, in this -case the node includes a `Heap Fetches:` statistic. - -### Index Scan - -A scan on an index that required retrieving some data from the table. - -### Bitmap Index Scan and Bitmap Heap scan - -Bitmap scans fall between sequential scans and index scans. These are typically -used when we would read too much data from an index scan, but too little to -perform a sequential scan. A bitmap scan uses what is known as a [bitmap -index](https://en.wikipedia.org/wiki/Bitmap_index) to perform its work. - -The [source code of PostgreSQL](https://gitlab.com/postgres/postgres/blob/REL_11_STABLE/src/include/nodes/plannodes.h#L441) -states the following on bitmap scans: - -> Bitmap Index Scan delivers a bitmap of potential tuple locations; it does not -> access the heap itself. The bitmap is used by an ancestor Bitmap Heap Scan -> node, possibly after passing through intermediate Bitmap And and/or Bitmap Or -> nodes to combine it with the results of other Bitmap Index Scans. - -### Limit - -Applies a `LIMIT` on the input rows. - -### Sort - -Sorts the input rows as specified using an `ORDER BY` statement. - -### Nested Loop - -A nested loop executes its child nodes for every row produced by a node that -precedes it. For example: - -```sql --> Nested Loop (cost=0.86..176.75 rows=87 width=0) (actual time=0.035..0.249 rows=36 loops=1) - Buffers: shared hit=155 - -> Index Only Scan using users_pkey on users users_1 (cost=0.43..4.95 rows=87 width=4) (actual time=0.029..0.123 rows=36 loops=1) - Index Cond: (id < 100) - Heap Fetches: 0 - -> Index Only Scan using users_pkey on users (cost=0.43..1.96 rows=1 width=4) (actual time=0.003..0.003 rows=1 loops=36) - Index Cond: (id = users_1.id) - Heap Fetches: 0 -``` - -Here the first child node (`Index Only Scan using users_pkey on users users_1`) -produces 36 rows, and is executed once (`rows=36 loops=1`). The next node -produces 1 row (`rows=1`), but is repeated 36 times (`loops=36`). This is -because the previous node produced 36 rows. - -This means that nested loops can quickly slow the query down if the various -child nodes keep producing many rows. - -## Optimising queries - -With that out of the way, let's see how we can optimise a query. Let's use the -following query as an example: - -```sql -SELECT COUNT(*) -FROM users -WHERE twitter != ''; -``` - -This query counts the number of users that have a Twitter profile set. -Let's run this using `EXPLAIN (ANALYZE, BUFFERS)`: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM users -WHERE twitter != ''; -``` - -This produces the following plan: - -```sql -Aggregate (cost=845110.21..845110.22 rows=1 width=8) (actual time=1271.157..1271.158 rows=1 loops=1) - Buffers: shared hit=202662 - -> Seq Scan on users (cost=0.00..844969.99 rows=56087 width=0) (actual time=0.019..1265.883 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487813 - Buffers: shared hit=202662 -Planning time: 0.390 ms -Execution time: 1271.180 ms -``` - -From this query plan we can see the following: - -1. We need to perform a sequential scan on the `users` table. -1. This sequential scan filters out 2,487,813 rows using a `Filter`. -1. We use 202,622 buffers, which equals 1.58 GB of memory. -1. It takes us 1.2 seconds to do all of this. - -Considering we are just counting users, that's quite expensive! - -Before we start making any changes, let's see if there are any existing indexes -on the `users` table that we might be able to use. We can obtain this -information by running `\d users` in a `psql` console, then scrolling down to -the `Indexes:` section: - -```sql -Indexes: - "users_pkey" PRIMARY KEY, btree (id) - "index_users_on_confirmation_token" UNIQUE, btree (confirmation_token) - "index_users_on_email" UNIQUE, btree (email) - "index_users_on_reset_password_token" UNIQUE, btree (reset_password_token) - "index_users_on_static_object_token" UNIQUE, btree (static_object_token) - "index_users_on_unlock_token" UNIQUE, btree (unlock_token) - "index_on_users_name_lower" btree (lower(name::text)) - "index_users_on_accepted_term_id" btree (accepted_term_id) - "index_users_on_admin" btree (admin) - "index_users_on_created_at" btree (created_at) - "index_users_on_email_trigram" gin (email gin_trgm_ops) - "index_users_on_feed_token" btree (feed_token) - "index_users_on_group_view" btree (group_view) - "index_users_on_incoming_email_token" btree (incoming_email_token) - "index_users_on_managing_group_id" btree (managing_group_id) - "index_users_on_name" btree (name) - "index_users_on_name_trigram" gin (name gin_trgm_ops) - "index_users_on_public_email" btree (public_email) WHERE public_email::text <> ''::text - "index_users_on_state" btree (state) - "index_users_on_state_and_user_type" btree (state, user_type) - "index_users_on_unconfirmed_email" btree (unconfirmed_email) WHERE unconfirmed_email IS NOT NULL - "index_users_on_user_type" btree (user_type) - "index_users_on_username" btree (username) - "index_users_on_username_trigram" gin (username gin_trgm_ops) - "tmp_idx_on_user_id_where_bio_is_filled" btree (id) WHERE COALESCE(bio, ''::character varying)::text IS DISTINCT FROM ''::text -``` - -Here we can see there is no index on the `twitter` column, which means -PostgreSQL has to perform a sequential scan in this case. Let's try to fix this -by adding the following index: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); -``` - -If we now re-run our query using `EXPLAIN (ANALYZE, BUFFERS)` we get the -following plan: - -```sql -Aggregate (cost=61002.82..61002.83 rows=1 width=8) (actual time=297.311..297.312 rows=1 loops=1) - Buffers: shared hit=51854 dirtied=19 - -> Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487830 - Heap Fetches: 26037 - Buffers: shared hit=51854 dirtied=19 -Planning time: 0.191 ms -Execution time: 297.334 ms -``` - -Now it takes just under 300 milliseconds to get our data, instead of 1.2 -seconds. However, we still use 51,854 buffers, which is about 400 MB of memory. -300 milliseconds is also quite slow for such a simple query. To understand why -this query is still expensive, let's take a look at the following: - -```sql -Index Only Scan using twitter_test on users (cost=0.43..60873.13 rows=51877 width=0) (actual time=279.184..293.532 rows=51833 loops=1) - Filter: ((twitter)::text <> ''::text) - Rows Removed by Filter: 2487830 -``` - -We start with an index only scan on our index, but we somehow still apply a -`Filter` that filters out 2,487,830 rows. Why is that? Well, let's look at how -we created the index: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter); -``` - -We told PostgreSQL to index all possible values of the `twitter` column, -even empty strings. Our query in turn uses `WHERE twitter != ''`. This means -that the index does improve things, as we don't need to do a sequential scan, -but we may still encounter empty strings. This means PostgreSQL _has_ to apply a -Filter on the index results to get rid of those values. - -Fortunately, we can improve this even further using "partial indexes". Partial -indexes are indexes with a `WHERE` condition that is applied when indexing data. -For example: - -```sql -CREATE INDEX CONCURRENTLY some_index ON users (email) WHERE id < 100 -``` - -This index would only index the `email` value of rows that match `WHERE id < -100`. We can use partial indexes to change our Twitter index to the following: - -```sql -CREATE INDEX CONCURRENTLY twitter_test ON users (twitter) WHERE twitter != ''; -``` - -After being created, if we run our query again we are given the following plan: - -```sql -Aggregate (cost=1608.26..1608.27 rows=1 width=8) (actual time=19.821..19.821 rows=1 loops=1) - Buffers: shared hit=44036 - -> Index Only Scan using twitter_test on users (cost=0.41..1479.71 rows=51420 width=0) (actual time=0.023..15.514 rows=51833 loops=1) - Heap Fetches: 1208 - Buffers: shared hit=44036 -Planning time: 0.123 ms -Execution time: 19.848 ms -``` - -That's _a lot_ better! Now it only takes 20 milliseconds to get the data, and we -only use about 344 MB of buffers (instead of the original 1.58 GB). The reason -this works is that now PostgreSQL no longer needs to apply a `Filter`, as the -index only contains `twitter` values that are not empty. - -Keep in mind that you shouldn't just add partial indexes every time you want to -optimise a query. Every index has to be updated for every write, and they may -require quite a bit of space, depending on the amount of indexed data. As a -result, first check if there are any existing indexes you may be able to reuse. -If there aren't any, check if you can perhaps slightly change an existing one to -fit both the existing and new queries. Only add a new index if none of the -existing indexes can be used in any way. - -When comparing execution plans, don't take timing as the only important metric. -Good timing is the main goal of any optimization, but it can be too volatile to -be used for comparison (for example, it depends a lot on the state of cache). -When optimizing a query, we usually need to reduce the amount of data we're -dealing with. Indexes are the way to work with fewer pages (buffers) to get the -result, so, during optimization, look at the number of buffers used (read and hit), -and work on reducing these numbers. Reduced timing is the consequence of reduced -buffer numbers. [Database Lab Engine](#database-lab-engine) guarantees that the plan is structurally -identical to production (and overall number of buffers is the same as on production), -but difference in cache state and I/O speed may lead to different timings. - -## Queries that can't be optimised - -Now that we have seen how to optimise a query, let's look at another query that -we might not be able to optimise: - -```sql -EXPLAIN (ANALYZE, BUFFERS) -SELECT COUNT(*) -FROM projects -WHERE visibility_level IN (0, 20); -``` - -The output of `EXPLAIN (ANALYZE, BUFFERS)` is as follows: - -```sql -Aggregate (cost=922420.60..922420.61 rows=1 width=8) (actual time=3428.535..3428.535 rows=1 loops=1) - Buffers: shared hit=208846 - -> Seq Scan on projects (cost=0.00..908053.18 rows=5746969 width=0) (actual time=0.041..2987.606 rows=5746940 loops=1) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 65677 - Buffers: shared hit=208846 -Planning time: 2.861 ms -Execution time: 3428.596 ms -``` - -Looking at the output we see the following Filter: - -```sql -Filter: (visibility_level = ANY ('{0,20}'::integer[])) -Rows Removed by Filter: 65677 -``` - -Looking at the number of rows removed by the filter, we may be tempted to add an -index on `projects.visibility_level` to somehow turn this Sequential scan + -filter into an index-only scan. - -Unfortunately, doing so is unlikely to improve anything. Contrary to what some -might believe, an index being present _does not guarantee_ that PostgreSQL -actually uses it. For example, when doing a `SELECT * FROM projects` it is much -cheaper to just scan the entire table, instead of using an index and then -fetching data from the table. In such cases PostgreSQL may decide to not use an -index. - -Second, let's think for a moment what our query does: it gets all projects with -visibility level 0 or 20. In the above plan we can see this produces quite a lot -of rows (5,745,940), but how much is that relative to the total? Let's find out -by running the following query: - -```sql -SELECT visibility_level, count(*) AS amount -FROM projects -GROUP BY visibility_level -ORDER BY visibility_level ASC; -``` - -For GitLab.com this produces: - -```sql - visibility_level | amount -------------------+--------- - 0 | 5071325 - 10 | 65678 - 20 | 674801 -``` - -Here the total number of projects is 5,811,804, and 5,746,126 of those are of -level 0 or 20. That's 98% of the entire table! - -So no matter what we do, this query retrieves 98% of the entire table. Since -most time is spent doing exactly that, there isn't really much we can do to -improve this query, other than _not_ running it at all. - -What is important here is that while some may recommend to straight up add an -index the moment you see a sequential scan, it is _much more important_ to first -understand what your query does, how much data it retrieves, and so on. After -all, you can not optimise something you do not understand. - -### Cardinality and selectivity - -Earlier we saw that our query had to retrieve 98% of the rows in the table. -There are two terms commonly used for databases: cardinality, and selectivity. -Cardinality refers to the number of unique values in a particular column in a -table. - -Selectivity is the number of unique values produced by an operation (for example, an -index scan or filter), relative to the total number of rows. The higher the -selectivity, the more likely PostgreSQL is able to use an index. - -In the above example, there are only 3 unique values: 0, 10, and 20. This means -the cardinality is 3. The selectivity in turn is also very low: 0.0000003% (2 / -5,811,804), because our `Filter` only filters using two values (`0` and `20`). -With such a low selectivity value it's not surprising that PostgreSQL decides -using an index is not worth it, because it would produce almost no unique rows. - -## Rewriting queries - -So the above query can't really be optimised as-is, or at least not much. But -what if we slightly change the purpose of it? What if instead of retrieving all -projects with `visibility_level` 0 or 20, we retrieve those that a user -interacted with somehow? - -Fortunately, GitLab has an answer for this, and it's a table called -`user_interacted_projects`. This table has the following schema: - -```sql -Table "public.user_interacted_projects" - Column | Type | Modifiers -------------+---------+----------- - user_id | integer | not null - project_id | integer | not null -Indexes: - "index_user_interacted_projects_on_project_id_and_user_id" UNIQUE, btree (project_id, user_id) - "index_user_interacted_projects_on_user_id" btree (user_id) -Foreign-key constraints: - "fk_rails_0894651f08" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE - "fk_rails_722ceba4f7" FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE -``` - -Let's rewrite our query to `JOIN` this table onto our projects, and get the -projects for a specific user: - -```sql -EXPLAIN ANALYZE -SELECT COUNT(*) -FROM projects -INNER JOIN user_interacted_projects ON user_interacted_projects.project_id = projects.id -WHERE projects.visibility_level IN (0, 20) -AND user_interacted_projects.user_id = 1; -``` - -What we do here is the following: - -1. Get our projects. -1. `INNER JOIN` `user_interacted_projects`, meaning we're only left with rows in - `projects` that have a corresponding row in `user_interacted_projects`. -1. Limit this to the projects with `visibility_level` of 0 or 20, and to - projects that the user with ID 1 interacted with. - -If we run this query we get the following plan: - -```sql - Aggregate (cost=871.03..871.04 rows=1 width=8) (actual time=9.763..9.763 rows=1 loops=1) - -> Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) - -> Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) - Index Cond: (user_id = 1) - -> Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) - Index Cond: (id = user_interacted_projects.project_id) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 0 - Planning time: 2.614 ms - Execution time: 9.809 ms -``` - -Here it only took us just under 10 milliseconds to get the data. We can also see -we're retrieving far fewer projects: - -```sql -Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) - Index Cond: (id = user_interacted_projects.project_id) - Filter: (visibility_level = ANY ('{0,20}'::integer[])) - Rows Removed by Filter: 0 -``` - -Here we see we perform 145 loops (`loops=145`), with every loop producing 1 row -(`rows=1`). This is much less than before, and our query performs much better! - -If we look at the plan we also see our costs are very low: - -```sql -Index Scan using projects_pkey on projects (cost=0.43..3.45 rows=1 width=4) (actual time=0.049..0.050 rows=1 loops=145) -``` - -Here our cost is only 3.45, and it takes us 7.25 milliseconds to do so (0.05 * 145). -The next index scan is a bit more expensive: - -```sql -Index Scan using index_user_interacted_projects_on_user_id on user_interacted_projects (cost=0.43..160.71 rows=205 width=4) (actual time=0.939..2.508 rows=145 loops=1) -``` - -Here the cost is 160.71 (`cost=0.43..160.71`), taking about 2.5 milliseconds -(based on the output of `actual time=....`). - -The most expensive part here is the "Nested Loop" that acts upon the result of -these two index scans: - -```sql -Nested Loop (cost=0.86..870.52 rows=203 width=0) (actual time=1.072..9.748 rows=143 loops=1) -``` - -Here we had to perform 870.52 disk page fetches for 203 rows, 9.748 -milliseconds, producing 143 rows in a single loop. - -The key takeaway here is that sometimes you have to rewrite (parts of) a query -to make it better. Sometimes that means having to slightly change your feature -to accommodate for better performance. - -## What makes a bad plan - -This is a bit of a difficult question to answer, because the definition of "bad" -is relative to the problem you are trying to solve. However, some patterns are -best avoided in most cases, such as: - -- Sequential scans on large tables -- Filters that remove a lot of rows -- Performing a certain step that requires _a lot_ of - buffers (for example, an index scan for GitLab.com that requires more than 512 MB). - -As a general guideline, aim for a query that: - -1. Takes no more than 10 milliseconds. Our target time spent in SQL per request - is around 100 milliseconds, so every query should be as fast as possible. -1. Does not use an excessive number of buffers, relative to the workload. For - example, retrieving ten rows shouldn't require 1 GB of buffers. -1. Does not spend a long amount of time performing disk IO operations. The - setting `track_io_timing` must be enabled for this data to be included in the - output of `EXPLAIN ANALYZE`. -1. Applies a `LIMIT` when retrieving rows without aggregating them, such as - `SELECT * FROM users`. -1. Doesn't use a `Filter` to filter out too many rows, especially if the query - does not use a `LIMIT` to limit the number of returned rows. Filters can - usually be removed by adding a (partial) index. - -These are _guidelines_ and not hard requirements, as different needs may require -different queries. The only _rule_ is that you _must always measure_ your query -(preferably using a production-like database) using `EXPLAIN (ANALYZE, BUFFERS)` -and related tools such as: - -- [`explain.depesz.com`](https://explain.depesz.com/). -- [`explain.dalibo.com/`](https://explain.dalibo.com/). - -## Producing query plans - -There are a few ways to get the output of a query plan. Of course you -can directly run the `EXPLAIN` query in the `psql` console, or you can -follow one of the other options below. - -### Database Lab Engine - -GitLab team members can use [Database Lab Engine](https://gitlab.com/postgres-ai/database-lab), and the companion -SQL optimization tool - [Joe Bot](https://gitlab.com/postgres-ai/joe). - -Database Lab Engine provides developers with their own clone of the production database, while Joe Bot helps with exploring execution plans. - -Joe Bot is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack, -and through its [web interface](https://console.postgres.ai/gitlab/joe-instances). - -With Joe Bot you can execute DDL statements (like creating indexes, tables, and columns) and get query plans for `SELECT`, `UPDATE`, and `DELETE` statements. - -For example, in order to test new index on a column that is not existing on production yet, you can do the following: - -Create the column: - -```sql -exec ALTER TABLE projects ADD COLUMN last_at timestamp without time zone -``` - -Create the index: - -```sql -exec CREATE INDEX index_projects_last_activity ON projects (last_activity_at) WHERE last_activity_at IS NOT NULL -``` - -Analyze the table to update its statistics: - -```sql -exec ANALYZE projects -``` - -Get the query plan: - -```sql -explain SELECT * FROM projects WHERE last_activity_at < CURRENT_DATE -``` - -Once done you can rollback your changes: - -```sql -reset -``` - -For more information about the available options, run: - -```sql -help -``` - -The web interface comes with the following execution plan visualizers included: - -- [Depesz](https://explain.depesz.com/) -- [PEV2](https://github.com/dalibo/pev2) -- [FlameGraph](https://github.com/mgartner/pg_flame) - -#### Tips & Tricks - -The database connection is now maintained during your whole session, so you can use `exec set ...` for any session variables (such as `enable_seqscan` or `work_mem`). These settings are applied to all subsequent commands until you reset them. For example you can disable parallel queries with - -```sql -exec SET max_parallel_workers_per_gather = 0 -``` - -### Rails console - -Using the [`activerecord-explain-analyze`](https://github.com/6/activerecord-explain-analyze) -you can directly generate the query plan from the Rails console: - -```ruby -pry(main)> require 'activerecord-explain-analyze' -=> true -pry(main)> Project.where('build_timeout > ?', 3600).explain(analyze: true) - Project Load (1.9ms) SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) - ↳ (pry):12 -=> EXPLAIN for: SELECT "projects".* FROM "projects" WHERE (build_timeout > 3600) -Seq Scan on public.projects (cost=0.00..2.17 rows=1 width=742) (actual time=0.040..0.041 rows=0 loops=1) - Output: id, name, path, description, created_at, updated_at, creator_id, namespace_id, ... - Filter: (projects.build_timeout > 3600) - Rows Removed by Filter: 14 - Buffers: shared hit=2 -Planning time: 0.411 ms -Execution time: 0.113 ms -``` - -### ChatOps - -[GitLab team members can also use our ChatOps solution, available in Slack using the -`/chatops` slash command](chatops_on_gitlabcom.md). - -NOTE: -While ChatOps is still available, the recommended way to generate execution plans is to use [Database Lab Engine](#database-lab-engine). - -You can use ChatOps to get a query plan by running the following: - -```sql -/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Visualising the plan using <https://explain.depesz.com/> is also supported: - -```sql -/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Quoting the query is not necessary. - -For more information about the available options, run: - -```sql -/chatops run explain --help -``` - -## Further reading - -A more extensive guide on understanding query plans can be found in -the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) -from [Dalibo.org](https://www.dalibo.com/en/). - -Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable/) dedicated to query plans. +<!-- This redirect file can be deleted after <2022-11-04>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md index d44f2f69168..5a5f987c37c 100644 --- a/doc/development/uploads/working_with_uploads.md +++ b/doc/development/uploads/working_with_uploads.md @@ -6,92 +6,295 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Uploads guide: Adding new uploads -Here, we describe how to add a new upload route [accelerated](index.md#workhorse-assisted-uploads) by Workhorse. - -Upload routes belong to one of these categories: - -1. Rails controllers: uploads handled by Rails controllers. -1. Grape API: uploads handled by a Grape API endpoint. -1. GraphQL API: uploads handled by a GraphQL resolve function. - -WARNING: -GraphQL uploads do not support [direct upload](index.md#direct-upload). Depending on the use case, the feature may not work on installations without NFS (like GitLab.com or Kubernetes installations). Uploading to object storage inside the GraphQL resolve function may result in timeout errors. For more details, follow [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). - -## Update Workhorse for the new route - -For both the Rails controller and Grape API uploads, Workhorse must be updated to get the -support for the new upload route. - -1. Open a new issue in the [Workhorse tracker](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/new) describing precisely the new upload route: - - The route's URL. - - The upload encoding. - - If possible, provide a dump of the upload request. -1. Implement and get the MR merged for this issue above. -1. Ask the Maintainers of [Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) to create a new release. You can do that in the merge request - directly during the maintainer review, or ask for it in the `#workhorse` Slack channel. -1. Bump the [Workhorse version file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) - to the version you have from the previous points, or bump it in the same merge request that contains - the Rails changes. Refer to [Implementing the new route with a Rails controller](#implementing-the-new-route-with-a-rails-controller) or [Implementing the new route with a Grape API endpoint](#implementing-the-new-route-with-a-grape-api-endpoint) below. - -## Implementing the new route with a Rails controller - -For a Rails controller upload, we usually have a `multipart/form-data` upload and there are a -few things to do: - -1. The upload is available under the parameter name you're using. For example, it could be an `artifact` - or a nested parameter such as `user[avatar]`. If you have the upload under the - `file` parameter, reading `params[:file]` should get you an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance. -1. Generally speaking, it's a good idea to check if the instance is from the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) class. For example, see how we checked -[that the parameter is indeed an `UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/commit/ea30fe8a71bf16ba07f1050ab4820607b5658719#51c0cc7a17b7f12c32bc41cfab3649ff2739b0eb_79_77). - -WARNING: -**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -instance using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` -passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -instance that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) -builds automatically for you. - -## Implementing the new route with a Grape API endpoint - -For a Grape API upload, we can have a body or multipart upload. Things are slightly more complicated: two endpoints are needed. One for the -Workhorse pre-upload authorization and one for accepting the upload metadata from Workhorse: - -1. Implement an endpoint with the URL + `/authorize` suffix that will: - - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the [API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). - - Check user permissions. - - Set the status to `200` with `status 200`. - - Set the content type with `content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE`. - - Use your dedicated `Uploader` class (let's say that it's `FileUploader`) to build the response with `FileUploader.workhorse_authorize(params)`. -1. Implement the endpoint for the upload request that will: - - Require all the `UploadedFile` objects as parameters. - - For example, if we expect a single parameter `file` to be an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) instance, -use `requires :file, type: ::API::Validations::Types::WorkhorseFile`. - - Body upload requests have their upload available under the parameter `file`. - - Check that the request is coming from Workhorse with the `require_gitlab_workhorse!` from the -[API helpers](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/api/helpers.rb). - - Check the user permissions. - - The remaining code of the processing. In this step, the code must read the parameter. For -our example, it would be `params[:file]`. - -WARNING: -**Do not** call `UploadedFile#from_params` directly! Do not build an [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -object using `UploadedFile#from_params`! This method can be unsafe to use depending on the `params` -passed. Instead, use the [`UploadedFile`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/uploaded_file.rb) -object that [`multipart.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb) -builds automatically for you. - -## Document Object Storage buckets and CarrierWave integration - -When using Object Storage, GitLab expects each kind of upload to maintain its own bucket in the respective -Object Storage destination. Moreover, the integration with CarrierWave is not used all the time. -The [Object Storage Working Group](https://about.gitlab.com/company/team/structure/working-groups/object-storage/) -is investigating an approach that unifies Object Storage buckets into a single one and removes CarrierWave -so as to simplify implementation and administration of uploads. - -Therefore, document new uploads here by slotting them into the following tables: - -- [Feature bucket details](#feature-bucket-details) -- [CarrierWave integration](#carrierwave-integration) +## Recommendations + +- When creating an uploader, [make it a subclass](#where-should-i-store-my-files) of `AttachmentUploader` +- Add your uploader to the [tables](#tables) in this document +- Do not add [new object storage buckets](#where-should-i-store-my-files) +- Implement [direct upload](#implementing-direct-upload-support) +- If you need to process your uploads, decide [where to do that](#processing-uploads) + +## Background information + +- [CarrierWave Uploaders](#carrierwave-uploaders) +- [GitLab modifications to CarrierWave](#gitlab-modifications-to-carrierwave) + +## Where should I store my files? + +CarrierWave Uploaders determine where files get +stored. When you create a new Uploader class you are deciding where to store the files of your new +feature. + +First of all, ask yourself if you need a new Uploader class. It is OK +to use the same Uploader class for different mountpoints or different +models. + +If you do want or need your own Uploader class then you should make it +a **subclass of `AttachmentUploader`**. You then inherit the storage +location and directory scheme from that class. The directory scheme +is: + +```ruby +File.join(model.class.underscore, mounted_as.to_s, model.id.to_s) +``` + +If you look around in the GitLab code base you will find quite a few +Uploaders that have their own storage location. For object storage, +this means Uploaders have their own buckets. We now **discourage** +adding new buckets for the following reasons: + +- Using a new bucket adds to development time because you need to make downstream changes in [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit), [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab) and [CNG](https://gitlab.com/gitlab-org/build/CNG). +- Using a new bucket requires GitLab.com Infrastructure changes, which slows down the roll-out of your new feature +- Using a new bucket slows down adoption of your new feature for self-managed GitLab installation: people cannot start using your new feature until their local GitLab administrator has configured the new bucket. + +By using an existing bucket you avoid all this extra work +and friction. The `Gitlab.config.uploads` storage location, which is what +`AttachmentUploader` uses, is guaranteed to already be configured. + +## Implementing Direct Upload support + +Below we will outline how to implement [direct upload](#direct-upload-via-workhorse) support. + +Using direct upload is not always necessary but it is usually a good +idea. Unless the uploads handled by your feature are both infrequent +and small, you probably want to implement direct upload. An example of +a feature with small and infrequent uploads is project avatars: these +rarely change and the application imposes strict size limits on them. + +If your feature handles uploads that are not both infrequent and small, +then not implementing direct upload support means that you are taking on +technical debt. At the very least, you should make sure that you _can_ +add direct upload support later. + +To support Direct Upload you need two things: + +1. A pre-authorization endpoint in Rails +1. A Workhorse routing rule + +Workhorse does not know where to store your upload. To find out it +makes a pre-authorization request. It also does not know whether or +where to make a pre-authorization request. For that you need the +routing rule. + +A note to those of us who remember, +[Workhorse used to be a separate project](https://gitlab.com/groups/gitlab-org/-/epics/4826): +it is not necessary anymore to break these two steps into separate merge +requests. In fact it is probably easier to do both in one merge +request. + +### Adding a Workhorse routing rule + +Routing rules are defined in +[workhorse/internal/upstream/routes.go](https://gitlab.com/gitlab-org/gitlab/-/blob/adf99b5327700cf34a845626481d7d6fcc454e57/workhorse/internal/upstream/routes.go). +They consist of: + +- An HTTP verb (usually "POST" or "PUT") +- A path regular expression +- An upload type: MIME multipart or "full request body" +- Optionally, you can also match on HTTP headers like `Content-Type` + +Example: + +```golang +u.route("PUT", apiProjectPattern+`packages/nuget/`, mimeMultipartUploader), +``` + +You should add a test for your routing rule to `TestAcceleratedUpload` +in +[workhorse/upload_test.go](https://gitlab.com/gitlab-org/gitlab/-/blob/adf99b5327700cf34a845626481d7d6fcc454e57/workhorse/upload_test.go). + +You should also manually verify that when you perform an upload +request for your new feature, Workhorse makes a pre-authorization +request. You can check this by looking at the Rails access logs. This +is necessary because if you make a mistake in your routing rule you +won't get a hard failure: you just end up using the less efficient +default path. + +### Adding a pre-authorization endpoint + +We distinguish three cases: Rails controllers, Grape API endpoints and +GraphQL resources. + +To start with the bad news: direct upload for GraphQL is currently not +supported. The reason for this is that Workhorse does not parse +GraphQL queries. Also see [issue #280819](https://gitlab.com/gitlab-org/gitlab/-/issues/280819). +Consider accepting your file upload via Grape instead. + +For Grape pre-authorization endpoints, look for existing examples that +implement `/authorize` routes. One example is the +[POST `:id/uploads/authorize` endpoint](https://gitlab.com/gitlab-org/gitlab/-/blob/9ad53d623eecebb799ce89eada951e4f4a59c116/lib/api/projects.rb#L642-651). +Note that this particular example is using FileUploader, which means +that the upload will be stored in the storage location (bucket) of +that Uploader class. + +For Rails endpoints you can use the +[WorkhorseAuthorization concern](https://gitlab.com/gitlab-org/gitlab/-/blob/adf99b5327700cf34a845626481d7d6fcc454e57/app/controllers/concerns/workhorse_authorization.rb). + +## Processing uploads + +Some features require us to process uploads, for example to extract +metadata from the uploaded file. There are a couple of different ways +you can implement this. The main choice is _where_ to implement the +processing, or "who is the processor". + +|Processor|Direct Upload possible?|Can reject HTTP request?|Implementation| +|---|---|---|---| +|Sidekiq|yes|no|Straightforward| +|Workhorse|yes|yes|Complex| +|Rails|no|yes|Easy| + +Processing in Rails looks appealing but it tends to lead to scaling +problems down the road because you cannot use direct upload. You are +then forced to rebuild your feature with processing in Workhorse. So +if the requirements of your feature allows it, doing the processing in +Sidekiq strikes a good balance between complexity and the ability to +scale. + +## CarrierWave Uploaders + +GitLab uses a modified version of +[CarrierWave](https://github.com/carrierwaveuploader/carrierwave) to +manage uploads. Below we will describe how we use CarrierWave and how +we modified it. + +The central concept of CarrierWave is the **Uploader** class. The +Uploader defines where files get stored, and optionally contains +validation and processing logic. To use an Uploader you must associate +it with a text column on an ActiveRecord model. This called "mounting" +and the column is called the "mountpoint". For example: + +```ruby +class Project < ApplicationRecord + mount_uploader :avatar, AttachmentUploader +end +``` + +Now if I upload an avatar called `tanuki.png` the idea is that in the +`projects.avatar` column for my project, CarrierWave stores the string +`tanuki.png`, and that the AttachmentUploader class contains the +configuration data and directory schema. For example if the project ID +is 123, the actual file may be in +`/var/opt/gitlab/gitlab-rails/uploads/-/system/project/avatar/123/tanuki.png`. +The directory +`/var/opt/gitlab/gitlab-rails/uploads/-/system/project/avatar/123/` +was chosen by the Uploader using among others configuration +(`/var/opt/gitlab/gitlab-rails/uploads`), the model name (`project`), +the model ID (`123`) and the mountpoint (`avatar`). + +> The Uploader determines the individual storage directory of your +> upload. The mountpoint column in your model contains the filename. + +You never access the mountpoint column directly because CarrierWave +defines a getter and setter on your model that operates on file handle +objects. + +### Optional Uploader behaviors + +Besides determining the storage directory for your upload, a +CarrierWave Uploader can implement several other behaviors via +callbacks. Not all of these behaviors are usable in GitLab. In +particular, you currently cannot use the `version` mechanism of +CarrierWave. Things you can do include: + +- Filename validation +- **Incompatible with direct upload:** One time pre-processing of file contents, e.g. image resizing +- **Incompatible with direct upload:** Encryption at rest + +Note that CarrierWave pre-processing behaviors such as image resizing +or encryption require local access to the uploaded file. This forces +you to upload the processed file from Ruby. This flies against direct +upload, which is all about _not_ doing the upload in Ruby. If you use +direct upload with an Uploader with pre-processing behaviors then the +pre-processing behaviors will be skipped silently. + +### CarrierWave Storage engines + +CarrierWave has 2 storage engines: + +|CarrierWave class|GitLab name|Description| +|---|---|---| +|`CarrierWave::Storage::File`|`ObjectStorage::Store::LOCAL` |Local files, accessed through the Ruby stdlib| +| `CarrierWave::Storage::Fog`|`ObjectStorage::Store::REMOTE`|Cloud files, accessed through the [Fog gem](https://github.com/fog/fog)| + +GitLab uses both of these engines, depending on configuration. + +The normal way to choose a storage engine in CarrierWave is to use the +`Uploader.storage` class method. In GitLab we do not do this; we have +overridden `Uploader#storage` instead. This allows us to vary the +storage engine file by file. + +### CarrierWave file lifecycle + +An Uploader is associated with two storage areas: regular storage and +cache storage. Each has its own storage engine. If you assign a file +to a mountpoint setter (`project.avatar = +File.open('/tmp/tanuki.png')`) you will copy/move the file to cache +storage as a side effect via the `cache!` method. To persist the file +you must somehow call the `store!` method. This either happens via +[ActiveRecord callbacks](https://github.com/carrierwaveuploader/carrierwave/blob/v1.3.2/lib/carrierwave/orm/activerecord.rb#L55) +or by calling `store!` on an Uploader instance. + +Normally you do not need to interact with `cache!` and `store!` but if +you need to debug GitLab CarrierWave modifications it is useful to +know that they are there and that they always get called. +Specifically, it is good to know that CarrierWave pre-processing +behaviors (`process` etc.) are implemented as `before :cache` hooks, +and in the case of direct upload, these hooks are ignored and do not +run. + +> Direct upload skips all CarrierWave `before :cache` hooks. + +## GitLab modifications to CarrierWave + +GitLab uses a modified version of CarrierWave to make a number of things possible. + +### Migrating data between storage engines + +In +[app/uploaders/object_storage.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/adf99b5327700cf34a845626481d7d6fcc454e57/app/uploaders/object_storage.rb) +there is code for migrating user data between local storage and object +storage. This code exists because for a long time, GitLab.com stored +uploads on local storage via NFS. This changed when as part of an infrastructure +migration we had to move the uploads to object storage. + +This is why the CarrierWave `storage` varies from upload to upload in +GitLab, and why we have database columns like `uploads.store` or +`ci_job_artifacts.file_store`. + +### Direct Upload via Workhorse + +Workhorse direct upload is a mechanism that lets us accept large +uploads without spending a lot of Ruby CPU time. Workhorse is written +in Go and goroutines have a much lower resource footprint than Ruby +threads. + +Direct upload works as follows. + +1. Workhorse accepts a user upload request +1. Workhorse pre-authenticates the request with Rails, and receives a temporary upload location +1. Workhorse stores the file upload in the user's request to the temporary upload location +1. Workhorse propagates the request to Rails +1. Rails issues a remote copy operation to copy the uploaded file from its temporary location to the final location +1. Rails deletes the temporary upload +1. Workhorse deletes the temporary upload a second time in case Rails timed out + +Normally, `cache!` returns an instance of +`CarrierWave::SanitizedFile`, and `store!` then +[uploads that file using Fog](https://github.com/carrierwaveuploader/carrierwave/blob/v1.3.2/lib/carrierwave/storage/fog.rb#L327-L335). + +In the case of object storage, with the modifications specific to GitLab, the +copying from the temporary location to the final location is +implemented by Rails fooling CarrierWave. When CarrierWave tries to +`cache!` the upload, we +[return](https://gitlab.com/gitlab-org/gitlab/-/blob/59b441d578e41cb177406a9799639e7a5aa9c7e1/app/uploaders/object_storage.rb#L367) +a `CarrierWave::Storage::Fog::File` file handle which points to the +temporary file. During the `store!` phase, CarrierWave then +[copies](https://github.com/carrierwaveuploader/carrierwave/blob/v1.3.2/lib/carrierwave/storage/fog.rb#L325) +this file to its intended location. + +## Tables + +The Scalability::Frameworks team is going to make object storage and uploads more easy to use and more robust. If you add or change uploaders, it helps us if you update this table too. This helps us keep an overview of where and how uploaders are used. ### Feature bucket details diff --git a/doc/development/utilities.md b/doc/development/utilities.md index b9b4c6448e2..3f6187a4c2e 100644 --- a/doc/development/utilities.md +++ b/doc/development/utilities.md @@ -188,6 +188,24 @@ Refer to [`strong_memoize.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/maste end ``` + Alternatively, use the `strong_memoize_attr` helper to memoize the method for you: + + ```ruby + class Find + include Gitlab::Utils::StrongMemoize + + def result + search + end + strong_memoize_attr :result + + strong_memoize_attr :enabled?, :enabled + def enabled? + Feature.enabled?(:some_feature) + end + end + ``` + - Clear memoization ```ruby diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index 55347edf4ec..0217eb96e5a 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -1,38 +1,11 @@ --- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'database/verifying_database_capabilities.md' +remove_date: '2022-11-06' --- -# Verifying Database Capabilities +This document was moved to [another location](database/verifying_database_capabilities.md). -Sometimes certain bits of code may only work on a certain database -version. While we try to avoid such code as much as possible sometimes it is -necessary to add database (version) specific behavior. - -To facilitate this we have the following methods that you can use: - -- `ApplicationRecord.database.version`: returns the PostgreSQL version number as a string - in the format `X.Y.Z`. - -This allows you to write code such as: - -```ruby -if ApplicationRecord.database.version.to_f >= 11.7 - run_really_fast_query -else - run_fast_query -end -``` - -## Read-only database - -The database can be used in read-only mode. In this case we have to -make sure all GET requests don't attempt any write operations to the -database. If one of those requests wants to write to the database, it needs -to be wrapped in a `Gitlab::Database.read_only?` or `Gitlab::Database.read_write?` -guard, to make sure it doesn't for read-only databases. - -We have a Rails Middleware that filters any potentially writing -operations (the `CUD` operations of CRUD) and prevent the user from trying -to update the database and getting a 500 error (see `Gitlab::Middleware::ReadOnly`). +<!-- This redirect file can be deleted after <2022-11-06>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/development/windows.md b/doc/development/windows.md index 3eed9c057ab..17dfaddef36 100644 --- a/doc/development/windows.md +++ b/doc/development/windows.md @@ -37,8 +37,7 @@ A list of software preinstalled on the Windows images is available at: [Preinsta ## GCP Windows image for development -The [shared Windows GitLab -runners](https://about.gitlab.com/releases/2020/01/22/gitlab-12-7-released/#windows-shared-runners-on-gitlabcom-beta) +The [shared Windows GitLab runners](https://about.gitlab.com/releases/2020/01/22/gitlab-12-7-released/#windows-shared-runners-on-gitlabcom-beta) are built with [Packer](https://www.packer.io/). The Infrastructure as Code repository for building the Google Cloud images is available at: diff --git a/doc/development/work_items.md b/doc/development/work_items.md index 9a17a152525..3625f85eb82 100644 --- a/doc/development/work_items.md +++ b/doc/development/work_items.md @@ -36,14 +36,12 @@ Here are some problems with current issues usage and why we are looking into wor differences in common interactions that the user needs to hold a complicated mental model of how they each behave. - Issues are not extensible enough to support all of the emerging jobs they need to facilitate. -- Codebase maintainability and feature development become bigger challenges as we grow the Issue type +- Codebase maintainability and feature development becomes a bigger challenge as we grow the Issue type. beyond its core role of issue tracking into supporting the different work item types and handling logic and structure differences. - New functionality is typically implemented with first class objects that import behavior from issues via shared concerns. This leads to duplicated effort and ultimately small differences between common interactions. This leads to inconsistent UX. -- Codebase maintainability and feature development becomes a bigger challenges as we grow issues - beyond its core role of issue tracking into supporting the different types and subtle differences between them. ## Work item terminology diff --git a/doc/development/workhorse/configuration.md b/doc/development/workhorse/configuration.md index b86bb824ea1..a94ba2b4fc6 100644 --- a/doc/development/workhorse/configuration.md +++ b/doc/development/workhorse/configuration.md @@ -211,7 +211,7 @@ Workhorse supports distributed tracing through [LabKit](https://gitlab.com/gitla using [OpenTracing APIs](https://opentracing.io). By default, no tracing implementation is linked into the binary. You can link in -different OpenTracing providers with [build tags](https://golang.org/pkg/go/build/#hdr-Build_Constraints) +different OpenTracing providers with [build tags](https://pkg.go.dev/go/build#hdr-Build_Constraints) or build constraints by setting the `BUILD_TAGS` make variable. For more details of the supported providers, refer to LabKit. For an example of @@ -234,7 +234,7 @@ When a user makes an HTTP request, such as creating a new project, the initial request is routed through Workhorse to another service, which may in turn, make other requests. To help trace the request as it flows across services, Workhorse generates a random value called a -[correlation ID](../../administration/troubleshooting/tracing_correlation_id.md). +[correlation ID](../../administration/logs/tracing_correlation_id.md). Workhorse sends this correlation ID via the `X-Request-Id` HTTP header. Some GitLab services, such as GitLab Shell, generate their own @@ -278,9 +278,9 @@ trusted_cidrs_for_x_forwarded_for = ["10.0.0.0/8", "127.0.0.1/32"] ## Continuous profiling Workhorse supports continuous profiling through [LabKit](https://gitlab.com/gitlab-org/labkit/) -using [Stackdriver Profiler](https://cloud.google.com/profiler). By default, the +using [Stackdriver Profiler](https://cloud.google.com/products/operations). By default, the Stackdriver Profiler implementation is linked in the binary using -[build tags](https://golang.org/pkg/go/build/#hdr-Build_Constraints), though it's not +[build tags](https://pkg.go.dev/go/build#hdr-Build_Constraints), though it's not required and can be skipped. For example: ```shell diff --git a/doc/development/workhorse/gitlab_features.md b/doc/development/workhorse/gitlab_features.md index 365cc7991d8..3b240d4cbc6 100644 --- a/doc/development/workhorse/gitlab_features.md +++ b/doc/development/workhorse/gitlab_features.md @@ -70,4 +70,4 @@ memory than it costs to have Workhorse look after it. - Workhorse does not clean up idle client connections. - We assume that all requests to Rails pass through Workhorse. -For more information see ['A brief history of GitLab Workhorse'](https://about.gitlab.com/2016/04/12/a-brief-history-of-gitlab-workhorse/). +For more information see ['A brief history of GitLab Workhorse'](https://about.gitlab.com/blog/2016/04/12/a-brief-history-of-gitlab-workhorse/). diff --git a/doc/development/workhorse/index.md b/doc/development/workhorse/index.md index 3aa7e945f53..962124248ef 100644 --- a/doc/development/workhorse/index.md +++ b/doc/development/workhorse/index.md @@ -10,8 +10,8 @@ GitLab Workhorse is a smart reverse proxy for GitLab. It handles "large" HTTP requests such as file downloads, file uploads, Git push/pull and Git archive downloads. -Workhorse itself is not a feature, but there are [several features in -GitLab](gitlab_features.md) that would not work efficiently without Workhorse. +Workhorse itself is not a feature, but there are +[several features in GitLab](gitlab_features.md) that would not work efficiently without Workhorse. The canonical source for Workhorse is [`gitlab-org/gitlab/workhorse`](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse). @@ -21,7 +21,7 @@ but that repository is no longer used for development. ## Install Workhorse -To install GitLab Workhorse you need [Go 1.15 or newer](https://golang.org/dl) and +To install GitLab Workhorse you need [Go 1.15 or newer](https://go.dev/dl) and [GNU Make](https://www.gnu.org/software/make/). To install into `/usr/local/bin` run `make install`. @@ -44,7 +44,7 @@ On some operating systems, such as FreeBSD, you may have to use ### Run time dependencies -Workhorse uses [ExifTool](https://www.sno.phy.queensu.ca/~phil/exiftool/) for +Workhorse uses [ExifTool](https://exiftool.org/) for removing EXIF data (which may contain sensitive information) from uploaded images. If you installed GitLab: diff --git a/doc/downgrade_ee_to_ce/index.md b/doc/downgrade_ee_to_ce/index.md index ca8d6f87809..2aa99eae084 100644 --- a/doc/downgrade_ee_to_ce/index.md +++ b/doc/downgrade_ee_to_ce/index.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Downgrading from EE to CE If you ever decide to downgrade your Enterprise Edition back to the -Community Edition, there are a few steps you need take beforehand. On Omnibus GitLab +Community Edition, there are a few steps you need to take beforehand. On Omnibus GitLab installations, these steps are made before installing the CE package on top of the current EE package. On installations from source, they are done before you change remotes and fetch the latest CE code. diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 65680227ad0..c6de723246c 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -391,7 +391,7 @@ git checkout <default-branch> git merge <feature-branch> ``` -In GitLab, you typically use a [merge request](../user/project/merge_requests/) to merge your changes, instead of using the command line. +In GitLab, you typically use a [merge request](../user/project/merge_requests/index.md) to merge your changes, instead of using the command line. To create a merge request from a fork to an upstream repository, see the [forking workflow](../user/project/repository/forking_workflow.md). diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index bc811cab3bf..b7a01cf61f4 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -32,15 +32,18 @@ Amazon provides a managed Kubernetes service offering known as [Amazon Elastic K ## Available Infrastructure as Code for GitLab Cloud Native Hybrid +The [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) is an effort made by GitLab to create a multi-cloud, multi-GitLab (Omnibus + Cloud Native Hybrid) toolkit to provision GitLab. GET is developed by GitLab developers and is open to community contributions. GET is where GitLab is investing its resources as the primary option for Infrastructure as Code, and is being actively used in production as a part of [GitLab Dedicated](../../subscriptions/gitlab_dedicated/index.md). + +Read the [GitLab Environment Toolkit (GET) direction](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md#direction) to learn more about the project and where it is going. + The [AWS Quick Start for GitLab Cloud Native Hybrid on EKS](https://aws-quickstart.github.io/quickstart-eks-gitlab/) is developed by AWS, GitLab, and the community that contributes to AWS Quick Starts, whether directly to the GitLab Quick Start or to the underlying Quick Start dependencies GitLab inherits (for example, EKS Quick Start). +GET is recommended for most deployments. The AWS Quick Start can be used if the IaC language of choice is CloudFormation, integration with AWS services like Control Tower is desired, or preference for a UI-driven configuration experience or when any aspect in the below table is an overriding concern. + NOTE: This automation is in **[Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta)**. GitLab is working with AWS on resolving [the outstanding issues](https://github.com/aws-quickstart/quickstart-eks-gitlab/issues?q=is%3Aissue+is%3Aopen+%5BHL%5D) before it is fully released. You can subscribe to this issue to be notified of progress and release announcements: [AWS Quick Start for GitLab Cloud Native Hybrid on EKS Status: Beta](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues/11).<br><br> The Beta version deploys Aurora PostgreSQL, but the release version will deploy Amazon RDS PostgreSQL due to [known issues](https://gitlab.com/gitlab-com/alliances/aws/public-tracker/-/issues?label_name%5B%5D=AWS+Known+Issue) with Aurora. All performance testing results will also be redone after this change has been made. -The [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/tree/main) is an effort made by GitLab to create a multi-cloud, multi-GitLab (Omnibus + Cloud Native Hybrid) toolkit to provision GitLab. GET is developed by GitLab developers and is open to community contributions. -It is helpful to review the [GitLab Environment Toolkit (GET) Issues](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/issues) to understand if any of them may affect your provisioning plans. - | | [AWS Quick Start for GitLab Cloud Native Hybrid on EKS](https://aws-quickstart.github.io/quickstart-eks-gitlab/) | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit) | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | | Overview and Vision | [AWS Quick Start](https://aws.amazon.com/quickstart/) | [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) | @@ -56,9 +59,11 @@ It is helpful to review the [GitLab Environment Toolkit (GET) Issues](https://gi | Results in a Ready-to-Use instance | Yes | Manual Actions or <br />Supplemental IaC Required | | **<u>Configuration Features</u>** | | | | Can deploy Omnibus GitLab (non-Kubernetes) | No | Yes | -| Results in a self-healing Gitaly Cluster configuration | Yes | No | +| Can deploy Single Instance Omnibus GitLab (non-Kubernetes) | No | Yes | | Complete Internal Encryption | 85%, Targeting 100% | Manual | | AWS GovCloud Support | Yes | TBD | +| No Code Form-Based Deployment User Experience Available | Yes | No | +| Full IaC User Experience Available | Yes | Yes | ### Two and Three Zone High Availability diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index 0d621217dd8..7f155a78140 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -233,8 +233,8 @@ The first thing that appears is the sign-in page. GitLab creates an administrato The credentials are: - Username: `root` -- Password: the password is automatically created, and there are [two ways to - find it](https://docs.bitnami.com/azure/faq/get-started/find-credentials/). +- Password: the password is automatically created, and there are + [two ways to find it](https://docs.bitnami.com/azure/faq/get-started/find-credentials/). After signing in, be sure to immediately [change the password](../../user/profile/index.md#change-your-password). diff --git a/doc/install/docker.md b/doc/install/docker.md index 356c025e168..7ec1b7c741a 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -166,7 +166,7 @@ install, and upgrade your Docker-based GitLab installation: GitLab: ```shell - docker-compose up -d + docker compose up -d ``` NOTE: @@ -510,8 +510,8 @@ To upgrade GitLab that was [installed using Docker Compose](#install-gitlab-usin 1. Download the newest release and upgrade your GitLab instance: ```shell - docker-compose pull - docker-compose up -d + docker compose pull + docker compose up -d ``` If you have used [tags](#use-tagged-versions-of-gitlab) instead, you'll need diff --git a/doc/install/installation.md b/doc/install/installation.md index 2f2ae016edd..2f6adb06322 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -129,8 +129,8 @@ sudo apt-get install libkrb5-dev ### Git -From GitLab 13.6, we recommend you use the [Git version provided by -Gitaly](https://gitlab.com/gitlab-org/gitaly/-/issues/2729) +From GitLab 13.6, we recommend you use the +[Git version provided by Gitaly](https://gitlab.com/gitlab-org/gitaly/-/issues/2729) that: - Is always at the version required by GitLab. @@ -239,8 +239,8 @@ sudo make install GitLab has several daemons written in Go. To install GitLab we need a Go compiler. The instructions below assume you use 64-bit -Linux. You can find downloads for other platforms at the [Go download -page](https://go.dev/dl). +Linux. You can find downloads for other platforms at the +[Go download page](https://go.dev/dl). ```shell # Remove former Go installation folder @@ -556,19 +556,6 @@ sudo -u git -H cp config/puma.rb.example config/puma.rb # cores you have available. You can get that number via the `nproc` command. sudo -u git -H editor config/puma.rb -# Disable 'git gc --auto' because GitLab already runs 'git gc' when needed -sudo -u git -H git config --global gc.auto 0 - -# Enable packfile bitmaps -sudo -u git -H git config --global repack.writeBitmaps true - -# Enable push options -# Refer to https://docs.gitlab.com/ee/user/project/push_options.html for more information. -sudo -u git -H git config --global receive.advertisePushOptions true - -# Enable fsyncObjectFiles to reduce risk of repository corruption if the server crashes -sudo -u git -H git config --global core.fsyncObjectFiles true - # Configure Redis connection settings sudo -u git -H cp config/resque.yml.example config/resque.yml sudo -u git -H cp config/cable.yml.example config/cable.yml @@ -1064,7 +1051,7 @@ See the [OmniAuth integration documentation](../integration/omniauth.md). ### Build your projects GitLab can build your projects. To enable that feature, you need runners to do that for you. -See the [GitLab Runner section](https://about.gitlab.com/stages-devops-lifecycle/continuous-integration/#gitlab-runner) to install it. +See the [GitLab Runner section](https://docs.gitlab.com/runner/) to install it. ### Adding your Trusted Proxies diff --git a/doc/install/next_steps.md b/doc/install/next_steps.md index 4defb62d254..4df2da875a6 100644 --- a/doc/install/next_steps.md +++ b/doc/install/next_steps.md @@ -28,7 +28,7 @@ installation. ## Security -- [Secure GitLab](../security/index.md#securing-your-gitlab-installation): +- [Secure GitLab](../security/index.md): Recommended practices to secure your GitLab instance. - Sign up for the GitLab [Security Newsletter](https://about.gitlab.com/company/preference-center/) to get notified for security updates upon release. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 27148e06ccb..93d66dc92a9 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -65,7 +65,7 @@ Because file system performance may affect the overall performance of GitLab, [we don't recommend using cloud-based file systems for storage](../administration/nfs.md#avoid-using-cloud-based-file-systems). NOTE: -[NFS for Git repository storage is deprecated](https://about.gitlab.com/releases/2021/06/22/gitlab-14-0-released/#nfs-for-git-repository-storage-deprecated). See our official [Statement of Support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for further information. +[NFS for Git repository storage is deprecated](https://about.gitlab.com/releases/2021/06/22/gitlab-14-0-released/#nfs-for-git-repository-storage-deprecated). See our official [Statement of Support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) for further information. ### CPU @@ -84,7 +84,7 @@ Memory requirements are dependent on the number of users and expected workload. The following is the recommended minimum Memory hardware guidance for a handful of example GitLab user base sizes. - **4GB RAM** is the **required** minimum memory size and supports up to 500 users - - Our [Memory Team](https://about.gitlab.com/handbook/engineering/development/enablement/memory/) is working to reduce the memory requirement. + - Our [Memory Team](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/memory/) is working to reduce the memory requirement. - 8GB RAM supports up to 1000 users - More users? Consult the [reference architectures page](../administration/reference_architectures/index.md) @@ -326,7 +326,7 @@ GitLab supports the following web browsers: - [Mozilla Firefox](https://www.mozilla.org/en-US/firefox/new/) - [Google Chrome](https://www.google.com/chrome/) -- [Chromium](https://www.chromium.org/getting-involved/dev-channel) +- [Chromium](https://www.chromium.org/getting-involved/dev-channel/) - [Apple Safari](https://www.apple.com/safari/) - [Microsoft Edge](https://www.microsoft.com/en-us/edge) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 5eac6ab7c84..dc3dc4d2012 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -35,6 +35,10 @@ before we remove them. |-----------------------|--------------------------| | GitLab 15.0 or later | OpenSearch 1.x or later | +If your version of Elasticsearch or OpenSearch is incompatible, to prevent data loss, indexing pauses and +a message is logged in the +[`elasticsearch.log`](../../administration/logs/index.md#elasticsearchlog) file. + If you are using a compatible version and after connecting to OpenSearch, you get the message `Elasticsearch version not compatible`, [unpause indexing](#unpause-indexing). ## System requirements @@ -53,7 +57,7 @@ each node should have: ## Install Elasticsearch Elasticsearch is *not* included in the Omnibus packages or when you install from -source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. +source. You must [install it separately](https://www.elastic.co/guide/en/elasticsearch/reference/7.16/install-elasticsearch.html "Elasticsearch 7.x installation documentation") and ensure you select your version. Detailed information on how to install Elasticsearch is out of the scope of this page. You can install Elasticsearch yourself, or use a cloud hosted offering such as [Elasticsearch Service](https://www.elastic.co/elasticsearch/service) (available on AWS, GCP, or Azure) or the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html) service. @@ -159,6 +163,12 @@ If you see an error such as `Permission denied - /home/git/gitlab-elasticsearch- may need to set the `production -> elasticsearch -> indexer_path` setting in your `gitlab.yml` file to `/usr/local/bin/gitlab-elasticsearch-indexer`, which is where the binary is installed. +### View indexing errors + +Errors from the [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer) are reported in +the [`sidekiq.log`](../../administration/logs/index.md#sidekiqlog) file with a `json.exception.class` of `Gitlab::Elastic::Indexer::Error`. +These errors may occur when indexing Git repository data. + ## Enable Advanced Search For GitLab instances with more than 50GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. @@ -212,7 +222,7 @@ The following Elasticsearch settings are available: | `Password` | The password of your Elasticsearch instance. | | `Number of Elasticsearch shards` | Elasticsearch indices are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indices with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). | | `Number of Elasticsearch replicas` | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value increases total disk space required by the index. | -| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). +| `Limit the number of namespaces and projects that can be indexed` | Enabling this allows you to select namespaces and projects to index. All other namespaces and projects use database search instead. If you enable this option but do not select any namespaces or projects, none are indexed. [Read more below](#limit-the-number-of-namespaces-and-projects-that-can-be-indexed).| | `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html), [AWS EC2 Instance Profile Credentials](https://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli), or [AWS ECS Tasks Credentials](https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-iam-roles.html). Please refer to [Identity and Access Management in Amazon OpenSearch Service](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html) for details of AWS hosted OpenSearch domain access policy configuration. | | `AWS Region` | The AWS region in which your OpenSearch Service is located. | | `AWS Access Key` | The AWS access key. | @@ -438,13 +448,13 @@ This should return something similar to: } ``` -In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog). +In order to debug issues with the migrations you can check the [`elasticsearch.log` file](../../administration/logs/index.md#elasticsearchlog). ### Retry a halted migration Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, it is halted and a notification is displayed in the Advanced Search integration settings. -It is recommended to check the [`elasticsearch.log` file](../../administration/logs.md#elasticsearchlog) to +It is recommended to check the [`elasticsearch.log` file](../../administration/logs/index.md#elasticsearchlog) to debug why the migration was halted and make any changes before retrying the migration. Once you believe you've fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried in the background. @@ -462,8 +472,7 @@ Before doing a major version GitLab upgrade, you should have completed all migrations that exist up until the latest minor version before that major version. If you have halted migrations, these need to be resolved and [retried](#retry-a-halted-migration) before proceeding with a major version -upgrade. Read more about [upgrading to a new major -version](../../update/index.md#upgrading-to-a-new-major-version). +upgrade. Read more about [upgrading to a new major version](../../update/index.md#upgrading-to-a-new-major-version). ## GitLab Advanced Search Rake tasks @@ -573,9 +582,9 @@ due to large volumes of data being indexed. WARNING: Indexing a large instance generates a lot of Sidekiq jobs. -Make sure to prepare for this task by having a [Scalable and Highly Available -Setup](../../administration/reference_architectures/index.md) or creating [extra -Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). +Make sure to prepare for this task by having a +[scalable setup](../../administration/reference_architectures/index.md) or creating +[extra Sidekiq processes](../../administration/sidekiq/extra_sidekiq_processes.md). 1. [Configure your Elasticsearch host and port](#enable-advanced-search). 1. Create empty indices: @@ -774,8 +783,8 @@ additional process dedicated to indexing a set of queues (or queue group). This ensure that indexing queues always have a dedicated worker, while the rest of the queues have another dedicated worker to avoid contention. -For this purpose, use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) -option that allows a more general selection of queue groups using a [worker matching query](../../administration/operations/extra_sidekiq_routing.md#worker-matching-query). +For this purpose, use the [queue selector](../../administration/sidekiq/extra_sidekiq_processes.md#queue-selector) +option that allows a more general selection of queue groups using a [worker matching query](../../administration/sidekiq/extra_sidekiq_routing.md#worker-matching-query). To handle these two queue groups, we generally recommend one of the following two options. You can either: @@ -809,7 +818,7 @@ WARNING: When starting multiple processes, the number of processes cannot exceed the number of CPU cores you want to dedicate to Sidekiq. Each Sidekiq process can use only one CPU core, subject to the available workload and concurrency settings. For more details, see how to -[run multiple Sidekiq processes](../../administration/operations/extra_sidekiq_processes.md). +[run multiple Sidekiq processes](../../administration/sidekiq/extra_sidekiq_processes.md). ### Two nodes, one process for each diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md index 97abf456baa..fb558441d6a 100644 --- a/doc/integration/advanced_search/elasticsearch_troubleshooting.md +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -14,16 +14,30 @@ Use the following information to troubleshoot Elasticsearch issues. One of the most valuable tools for identifying issues with the Elasticsearch integration are logs. The most relevant logs for this integration are: -1. [`sidekiq.log`](../../administration/logs.md#sidekiqlog) - All of the +1. [`sidekiq.log`](../../administration/logs/index.md#sidekiqlog) - All of the indexing happens in Sidekiq, so much of the relevant logs for the Elasticsearch integration can be found in this file. -1. [`elasticsearch.log`](../../administration/logs.md#elasticsearchlog) - There +1. [`elasticsearch.log`](../../administration/logs/index.md#elasticsearchlog) - There are additional logs specific to Elasticsearch that are sent to this file that may contain useful diagnostic information about searching, indexing or migrations. Here are some common pitfalls and how to overcome them. +## Common terminology + +- **Lucene**: A full-text search library written in Java. +- **Near real time (NRT)**: Refers to the slight latency from the time to index a + document to the time when it becomes searchable. +- **Cluster**: A collection of one or more nodes that work together to hold all + the data, providing indexing and search capabilities. +- **Node**: A single server that works as part of a cluster. +- **Index**: A collection of documents that have somewhat similar characteristics. +- **Document**: A basic unit of information that can be indexed. +- **Shards**: Fully-functional and independent subdivisions of indices. Each shard is actually + a Lucene index. +- **Replicas**: Failover mechanisms that duplicate indices. + ## How can I verify that my GitLab instance is using Elasticsearch? There are a couple of ways to achieve that: @@ -44,6 +58,20 @@ There are a couple of ways to achieve that: ::Gitlab::CurrentSettings.elasticsearch_limit_indexing? # Whether or not Elasticsearch is limited only to certain projects/namespaces ``` +- Confirm searches use Elasticsearch by accessing the [rails console] + (../../administration/operations/rails_console.md) and running the following commands: + + ```rails + u = User.find_by_email('email_of_user_doing_search') + s = SearchService.new(u, {:search => 'search_term'}) + pp s.search_objects.class + ``` + + The output from the last command is the key here. If it shows: + + - `ActiveRecord::Relation`, **it is not** using Elasticsearch. + - `Kaminari::PaginatableArray`, **it is** using Elasticsearch. + - If Elasticsearch is limited to specific namespaces and you need to know if Elasticsearch is being used for a specific project or namespace, you can use the Rails console: @@ -53,13 +81,57 @@ There are a couple of ways to achieve that: ::Gitlab::CurrentSettings.search_using_elasticsearch?(scope: Project.find_by_full_path("/my-namespace/my-project")) ``` -## I updated GitLab and now I can't find anything +## Troubleshooting indexing + +Troubleshooting indexing issues can be tricky. It can pretty quickly go to either GitLab +support or your Elasticsearch administrator. + +The best place to start is to determine if the issue is with creating an empty index. +If it is, check on the Elasticsearch side to determine if the `gitlab-production` (the +name for the GitLab index) exists. If it exists, manually delete it on the Elasticsearch +side and attempt to recreate it from the +[`recreate_index`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) +Rake task. + +If you still encounter issues, try creating an index manually on the Elasticsearch +instance. The details of the index aren't important here, as we want to test if indices +can be made. If the indices: + +- Cannot be made, speak with your Elasticsearch administrator. +- Can be made, Escalate this to GitLab support. + +If the issue is not with creating an empty index, the next step is to check for errors +during the indexing of projects. If errors do occur, they stem from either the indexing: + +- On the GitLab side. You need to rectify those. If they are not + something you are familiar with, contact GitLab support for guidance. +- Within the Elasticsearch instance itself. See if the error is [documented and has a fix](../../integration/advanced_search/elasticsearch_troubleshooting.md). If not, speak with your Elasticsearch administrator. + +If the indexing process does not present errors, check the status of the indexed projects. You can do this via the following Rake tasks: + +- [`sudo gitlab-rake gitlab:elastic:index_projects_status`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows the overall status) +- [`sudo gitlab-rake gitlab:elastic:projects_not_indexed`](../../integration/advanced_search/elasticsearch.md#gitlab-advanced-search-rake-tasks) (shows specific projects that are not indexed) + +If: + +- Everything is showing at 100%, escalate to GitLab support. This could be a potential + bug/issue. +- You do see something not at 100%, attempt to reindex that project. To do this, + run `sudo gitlab-rake gitlab:elastic:index_projects ID_FROM=<project ID> ID_TO=<project ID>`. + +If reindexing the project shows: + +- Errors on the GitLab side, escalate those to GitLab support. +- Elasticsearch errors or doesn't present any errors at all, reach out to your + Elasticsearch administrator to check the instance. + +### I updated GitLab and now I can't find anything We continuously make updates to our indexing strategies and aim to support newer versions of Elasticsearch. When indexing changes are made, it may be necessary for you to [reindex](elasticsearch.md#zero-downtime-reindexing) after updating GitLab. -## I indexed all the repositories but I can't get any hits for my search term in the UI +### I indexed all the repositories but I can't get any hits for my search term in the UI Make sure you [indexed all the database data](elasticsearch.md#enable-advanced-search). @@ -79,26 +151,29 @@ curl --request GET <elasticsearch_server_ip>:9200/gitlab-production/_search?q=<s More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-filter-context.html) are also possible. -It is important to understand at which level the problem is manifesting (UI, Rails code, Elasticsearch side) to be able to [troubleshoot further](../../administration/troubleshooting/elasticsearch.md#search-results-workflow). +If the results: + +- Sync up, please check that you are using [supported syntax](../../user/search/global_search/advanced_search_syntax.md). Note that Advanced Search does not support [exact substring matching](https://gitlab.com/gitlab-org/gitlab/-/issues/325234). +- Do not match up, this indicates a problem with the documents generated from the project. It is best to [re-index that project](../advanced_search/elasticsearch.md#indexing-a-range-of-projects-or-a-specific-project) NOTE: The above instructions are not to be used for scenarios that only index a [subset of namespaces](elasticsearch.md#limit-the-number-of-namespaces-and-projects-that-can-be-indexed). See [Elasticsearch Index Scopes](elasticsearch.md#advanced-search-index-scopes) for more information on searching for specific types of data. -## I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything +### I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything You must re-run all the Rake tasks to reindex the database, repositories, and wikis. -## The indexing process is taking a very long time +### The indexing process is taking a very long time The more data present in your GitLab instance, the longer the indexing process takes. -## There are some projects that weren't indexed, but I don't know which ones +### There are some projects that weren't indexed, but I don't know which ones You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. -## No new data is added to the Elasticsearch index when I push code +### No new data is added to the Elasticsearch index when I push code NOTE: This was [fixed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35936) in GitLab 13.2 and the Rake task is not available for versions greater than that. @@ -109,6 +184,116 @@ When performing the initial indexing of blobs, we lock all projects until the pr sudo gitlab-rake gitlab:elastic:clear_locked_projects ``` +### Indexing fails with `error: elastic: Error 429 (Too Many Requests)` + +If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: + +- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. +- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/sidekiq/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. + +### Indexing is very slow or fails with `rejected execution of coordinating operation` messages + +Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. +Ensure that your Elasticsearch cluster meets the [system requirements](elasticsearch.md#system-requirements) and has enough resources +to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). + +### Last resort to recreate an index + +There may be cases where somehow data never got indexed and it's not in the +queue, or the index is somehow in a state where migrations just cannot +proceed. It is always best to try to troubleshoot the root cause of the problem +by [viewing the logs](#view-logs). + +If there are no other options, then you always have the option of recreating the +entire index from scratch. If you have a small GitLab installation, this can +sometimes be a quick way to resolve a problem, but if you have a large GitLab +installation, then this might take a very long time to complete. Until the +index is fully recreated, your index does not serve correct search results, +so you may want to disable **Search with Elasticsearch** while it is running. + +If you are sure you've read the above caveats and want to proceed, then you +should run the following Rake task to recreate the entire index from scratch: + +**For Omnibus installations** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +sudo gitlab-rake gitlab:elastic:index +``` + +**For installations from source** + +```shell +# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE +cd /home/git/gitlab +sudo -u git -H bundle exec rake gitlab:elastic:index +``` + +### Troubleshooting performance + +Troubleshooting performance can be difficult on Elasticsearch. There is a ton of tuning +that *can* be done, but the majority of this falls on shoulders of a skilled +Elasticsearch administrator. + +Generally speaking, ensure: + +- The Elasticsearch server **is not** running on the same node as GitLab. +- The Elasticsearch server have enough RAM and CPU cores. +- That sharding **is** being used. + +Going into some more detail here, if Elasticsearch is running on the same server as GitLab, resource contention is **very** likely to occur. Ideally, Elasticsearch, which requires ample resources, should be running on its own server (maybe coupled with Logstash and Kibana). + +When it comes to Elasticsearch, RAM is the key resource. Elasticsearch themselves recommend: + +- **At least** 8 GB of RAM for a non-production instance. +- **At least** 16 GB of RAM for a production instance. +- Ideally, 64 GB of RAM. + +For CPU, Elasticsearch recommends at least 2 CPU cores, but Elasticsearch states common +setups use up to 8 cores. For more details on server specs, check out +[Elasticsearch's hardware guide](https://www.elastic.co/guide/en/elasticsearch/guide/current/hardware.html). + +Beyond the obvious, sharding comes into play. Sharding is a core part of Elasticsearch. +It allows for horizontal scaling of indices, which is helpful when you are dealing with +a large amount of data. + +With the way GitLab does indexing, there is a **huge** amount of documents being +indexed. By utilizing sharding, you can speed up Elasticsearch's ability to locate +data, since each shard is a Lucene index. + +If you are not using sharding, you are likely to hit issues when you start using +Elasticsearch in a production environment. + +Keep in mind that an index with only one shard has **no scale factor** and will +likely encounter issues when called upon with some frequency. + +If you need to know how many shards, read +[Elasticsearch's documentation on capacity planning](https://www.elastic.co/guide/en/elasticsearch/guide/2.x/capacity-planning.html), +as the answer is not straight forward. + +The easiest way to determine if sharding is in use is to check the output of the +[Elasticsearch Health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html): + +- Red means the cluster is down. +- Yellow means it is up with no sharding/replication. +- Green means it is healthy (up, sharding, replicating). + +For production use, it should always be green. + +Beyond these steps, you get into some of the more complicated things to check, +such as merges and caching. These can get complicated and it takes some time to +learn them, so it is best to escalate/pair with an Elasticsearch expert if you need to +dig further into these. + +Feel free to reach out to GitLab support, but this is likely to be something a skilled +Elasticsearch administrator has more experience with. + +## Issues with migrations + +Please ensure you've read about [Elasticsearch Migrations](../advanced_search/elasticsearch.md#advanced-search-migrations). + +If there is a halted migration and your [`elasticsearch.log`](../../administration/logs/index.md#elasticsearchlog) file contain errors, this could potentially be a bug/issue. Escalate to GitLab support if retrying migrations does not succeed. + ## `Can't specify parent if no parent field has been configured` error If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indices, you get @@ -146,6 +331,10 @@ This exception is seen when your Elasticsearch cluster is configured to reject r AWS has [fixed limits](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#network-limits) for this setting ("Maximum size of HTTP request payloads"), based on the size of the underlying instance. +## `Faraday::TimeoutError (execution expired)` error when using a proxy + +Set a custom `gitlab_rails['env']` environment variable, called [`no_proxy`](https://docs.gitlab.com/omnibus/settings/environment-variables.html) with the IP address of your Elasticsearch host. + ## My single node Elasticsearch cluster status never goes from `yellow` to `green` even though everything seems to be running properly **For a single node Elasticsearch cluster the functional cluster health status is yellow** (never green) because the primary shard is allocated but replicas cannot be as there is no other node to which Elasticsearch can assign a replica. This also applies if you are using the [Amazon OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/aes-handling-errors.html#aes-handling-errors-yellow-cluster-status) service. @@ -182,10 +371,6 @@ reason may be incompatible with our integration. You should try disabling plugins so you can rule out the possibility that the plugin is causing the problem. -## Low-level troubleshooting - -There is a [more structured, lower-level troubleshooting document](../../administration/troubleshooting/elasticsearch.md) for when you experience other issues, including poor performance. - ## Elasticsearch `code_analyzer` doesn't account for all code cases The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration. @@ -196,38 +381,6 @@ Improvements to the `code_analyzer` pattern and filters are being discussed in [ In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. -## Last resort to recreate an index - -There may be cases where somehow data never got indexed and it's not in the -queue, or the index is somehow in a state where migrations just cannot -proceed. It is always best to try to troubleshoot the root cause of the problem -by [viewing the logs](#view-logs). - -If there are no other options, then you always have the option of recreating the -entire index from scratch. If you have a small GitLab installation, this can -sometimes be a quick way to resolve a problem, but if you have a large GitLab -installation, then this might take a very long time to complete. Until the -index is fully recreated, your index does not serve correct search results, -so you may want to disable **Search with Elasticsearch** while it is running. - -If you are sure you've read the above caveats and want to proceed, then you -should run the following Rake task to recreate the entire index from scratch: - -**For Omnibus installations** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -sudo gitlab-rake gitlab:elastic:index -``` - -**For installations from source** - -```shell -# WARNING: DO NOT RUN THIS UNTIL YOU READ THE DESCRIPTION ABOVE -cd /home/git/gitlab -sudo -u git -H bundle exec rake gitlab:elastic:index -``` - ## How does Advanced Search handle private projects? Advanced Search stores all the projects in the same Elasticsearch indices, @@ -235,19 +388,6 @@ however, searches only surface results that can be viewed by the user. Advanced Search honors all permission checks in the application by filtering out projects that a user does not have access to at search time. -## Indexing fails with `error: elastic: Error 429 (Too Many Requests)` - -If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: - -- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/operations/extra_sidekiq_processes.md#queue-selector) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. - -## Indexing is very slow or fails with `rejected execution of coordinating operation` messages - -Bulk requests getting rejected by the Elasticsearch nodes are likely due to load and lack of available memory. -Ensure that your Elasticsearch cluster meets the [system requirements](elasticsearch.md#system-requirements) and has enough resources -to perform bulk operations. See also the error ["429 (Too Many Requests)"](#indexing-fails-with-error-elastic-error-429-too-many-requests). - ## Access requirements for the self-managed AWS OpenSearch Service To use the self-managed AWS OpenSearch Service with GitLab, configure your instance's domain access policies diff --git a/doc/integration/azure.md b/doc/integration/azure.md index 515e7406545..da1aa574bd6 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -107,6 +107,24 @@ Alternatively, add the `User.Read.All` application permission. ] ``` + For [alternative Azure clouds](https://docs.microsoft.com/en-us/azure/active-directory/develop/authentication-national-cloud), + configure `base_azure_url` under the `args` section. For example, for Azure Government Community Cloud (GCC): + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "azure_activedirectory_v2", + "label" => "Provider name", # optional label for login button, defaults to "Azure AD v2" + "args" => { + "client_id" => "CLIENT ID", + "client_secret" => "CLIENT SECRET", + "tenant_id" => "TENANT ID", + "base_azure_url" => "https://login.microsoftonline.us" + } + } + ] + ``` + - **For installations from source** For the v1.0 endpoint: @@ -115,8 +133,8 @@ Alternatively, add the `User.Read.All` application permission. - { name: 'azure_oauth2', # label: 'Provider name', # optional label for login button, defaults to "Azure AD" args: { client_id: 'CLIENT ID', - client_secret: 'CLIENT SECRET', - tenant_id: 'TENANT ID' } } + client_secret: 'CLIENT SECRET', + tenant_id: 'TENANT ID' } } ``` For the v2.0 endpoint: @@ -125,14 +143,25 @@ Alternatively, add the `User.Read.All` application permission. - { name: 'azure_activedirectory_v2', label: 'Provider name', # optional label for login button, defaults to "Azure AD v2" args: { client_id: "CLIENT ID", - client_secret: "CLIENT SECRET", - tenant_id: "TENANT ID" } } + client_secret: "CLIENT SECRET", + tenant_id: "TENANT ID" } } + ``` + + For [alternative Azure clouds](https://docs.microsoft.com/en-us/azure/active-directory/develop/authentication-national-cloud), + configure `base_azure_url` under the `args` section. For example, for Azure Government Community Cloud (GCC): + + ```yaml + - { name: 'azure_activedirectory_v2', + label: 'Provider name', # optional label for login button, defaults to "Azure AD v2" + args: { client_id: "CLIENT ID", + client_secret: "CLIENT SECRET", + tenant_id: "TENANT ID", + base_azure_url: "https://login.microsoftonline.us" } } ``` - You can optionally add the following parameters: + In addition, you can optionally add the following parameters to the `args` section: - - `base_azure_url` for different locales. For example, `base_azure_url: "https://login.microsoftonline.de"`. - - `scope`, which you add to `args`. The default is `openid profile email`. + - `scope` for [OAuth2 scopes](https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-auth-code-flow). The default is `openid profile email`. 1. Save the configuration file. diff --git a/doc/integration/cas.md b/doc/integration/cas.md index a0cb6bd98cd..38305967246 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -4,7 +4,11 @@ group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# CAS OmniAuth Provider **(FREE SELF)** +# CAS OmniAuth provider (deprecated) **(FREE SELF)** + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369127) in GitLab 15.3 and is planned for +removal in 16.0. To enable the CAS OmniAuth provider you must register your application with your CAS instance. This requires the service URL GitLab supplies to CAS. It should be diff --git a/doc/integration/datadog.md b/doc/integration/datadog.md index a9be7754cb9..b8624545c41 100644 --- a/doc/integration/datadog.md +++ b/doc/integration/datadog.md @@ -33,8 +33,7 @@ project, group, or instance level: 1. Select **Active** to enable the integration. 1. Specify the [**Datadog site**](https://docs.datadoghq.com/getting_started/site/) to send data to. 1. Provide your Datadog **API key**. -<!-- 1. Optional. Select **Enable logs collection** to enable logs collection for the output of jobs. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346339) in GitLab 14.8.) --> -<!-- TODO: uncomment the archive_trace_events field once :datadog_integration_logs_collection is rolled out. Rollout issue: https://gitlab.com/gitlab-org/gitlab/-/issues/346339 --> +1. Optional. Select **Enable logs collection** to enable logs collection for the output of jobs. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346339) in GitLab 15.3.) 1. Optional. To override the API URL used to send data directly, provide an **API URL**. Used only in advanced scenarios. 1. Optional. If you use more than one GitLab instance, provide a unique **Service** name diff --git a/doc/integration/github.md b/doc/integration/github.md index 3011155f825..ad90c714dac 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -157,13 +157,29 @@ To fix this issue, you must disable SSL verification: 1. Change the global Git `sslVerify` option to `false` on the GitLab server. - - **For Omnibus installations** + - **For Omnibus installations in [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later**: + + ```ruby + gitaly['gitconfig'] = [ + {key: "http.sslVerify", value: "false"}, + ] + ``` + + - **For Omnibus installations in GitLab 15.2 and earlier (legacy method)**: ```ruby omnibus_gitconfig['system'] = { "http" => ["sslVerify = false"] } ``` - - **For installations from source** + - **For installations from source in [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later**, edit the Gitaly configuration (`gitaly.toml`): + + ```toml + [[git.config]] + key = "http.sslVerify" + value = "false" + ``` + + - **For installations from source in GitLab 15.2 and earlier (legacy method)**: ```shell git config --global http.sslVerify false @@ -180,7 +196,7 @@ GitLab instance and GitHub Enterprise. To check for a connectivity issue: -1. Go to the [`production.log`](../administration/logs.md#productionlog) +1. Go to the [`production.log`](../administration/logs/index.md#productionlog) on your GitLab server and look for the following error: ``` plaintext diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 02705d9dec3..fee1e573384 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -77,7 +77,7 @@ GitLab.com generates an application ID and secret key for you to use. app_id: "YOUR_APP_ID", app_secret: "YOUR_APP_SECRET", args: { scope: "read_user" # optional: defaults to the scopes of the application - , client_options: { site: "https://gitlab.example.com/api/v4" } } + , client_options: { site: "https://gitlab.example.com" } } } ] ``` @@ -98,9 +98,13 @@ GitLab.com generates an application ID and secret key for you to use. label: 'Provider name', # optional label for login button, defaults to "GitLab.com" app_id: 'YOUR_APP_ID', app_secret: 'YOUR_APP_SECRET', - args: { "client_options": { "site": 'https://gitlab.example.com/api/v4' } } + args: { "client_options": { "site": 'https://gitlab.example.com' } } ``` + NOTE: + In GitLab 15.1 and earlier, the `site` parameter requires an `/api/v4` suffix. + We recommend you drop this suffix after you upgrade to GitLab 15.2 or later. + 1. Change `'YOUR_APP_ID'` to the Application ID from the GitLab.com application page. 1. Change `'YOUR_APP_SECRET'` to the secret from the GitLab.com application page. 1. Save the configuration file. diff --git a/doc/integration/img/jenkins_gitlab_service.png b/doc/integration/img/jenkins_gitlab_service.png Binary files differdeleted file mode 100644 index 682a5ae8ee2..00000000000 --- a/doc/integration/img/jenkins_gitlab_service.png +++ /dev/null diff --git a/doc/integration/img/jenkins_project.png b/doc/integration/img/jenkins_project.png Binary files differdeleted file mode 100644 index 126b05c8879..00000000000 --- a/doc/integration/img/jenkins_project.png +++ /dev/null diff --git a/doc/integration/img/omniauth_providers_v_14_6.png b/doc/integration/img/omniauth_providers_v_14_6.png Binary files differdeleted file mode 100644 index b434e9a210b..00000000000 --- a/doc/integration/img/omniauth_providers_v_14_6.png +++ /dev/null diff --git a/doc/integration/index.md b/doc/integration/index.md index 85ebac5b40c..f5b088b47f7 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -20,7 +20,6 @@ GitLab can be configured to authenticate access requests with the following auth - Enable the [Auth0 OmniAuth](auth0.md) provider. - Enable sign in with [Bitbucket](bitbucket.md) accounts. -- Configure GitLab to sign in using [CAS](cas.md). - Integrate with [Kerberos](kerberos.md). - Enable sign in via [LDAP](../administration/auth/ldap/index.md). - Enable [OAuth2 provider](oauth_provider.md) application creation. diff --git a/doc/integration/jenkins_deprecated.md b/doc/integration/jenkins_deprecated.md index 57219b18047..5010545b73a 100644 --- a/doc/integration/jenkins_deprecated.md +++ b/doc/integration/jenkins_deprecated.md @@ -2,62 +2,12 @@ stage: Ecosystem group: Integrations info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +remove_date: '2022-10-29' +redirect_to: 'jenkins.md' --- -# Jenkins CI (deprecated) service **(FREE)** +# Jenkins CI service (removed) **(FREE)** -NOTE: -In GitLab 8.3, Jenkins integration using the -[GitLab Hook Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -was deprecated in favor of the -[GitLab Plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Plugin). -Please use documentation for the new [Jenkins CI service](jenkins.md). - -NOTE: -This service was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1600) in GitLab 13.0 - -Integration includes: - -- Trigger Jenkins build after push to repository -- Show build status on Merge request page - -Requirements: - -- [Jenkins GitLab Hook plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -- Git clone access for Jenkins from GitLab repository (via SSH key) - -## Jenkins - -1. Install [GitLab Hook plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -1. Set up Jenkins project - - - -## GitLab - -In GitLab, perform the following steps. - -### Read access to repository - -Jenkins needs read access to the GitLab repository. We already specified a -private key to use in Jenkins, now we must add a public one to the GitLab -project. For that case we need a Deploy key. Read the documentation on -[how to set up a Deploy key](../user/project/deploy_keys/index.md). - -### Jenkins service - -Now navigate to GitLab services page and activate Jenkins - - - -Done! When you push to GitLab, it creates a build for Jenkins. You can view the -merge request build status with a link to the Jenkins build. - -### Multi-project Configuration - -The GitLab Hook plugin in Jenkins supports the automatic creation of a project -for each feature branch. After configuration GitLab triggers feature branch -builds and a corresponding project is created in Jenkins. - -Configure the GitLab Hook plugin in Jenkins. Go to 'Manage Jenkins' and then -'Configure System'. Find the 'GitLab Web Hook' section and configure as shown below. +This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/1600) +in GitLab 13.0. +Use the [Jenkins integration](jenkins.md) instead. diff --git a/doc/integration/jira/dvcs.md b/doc/integration/jira/dvcs.md index d35c21f6187..43a5349e0e5 100644 --- a/doc/integration/jira/dvcs.md +++ b/doc/integration/jira/dvcs.md @@ -261,7 +261,7 @@ resynchronize the information: - To complete a *full sync*, press `Shift` and select the sync icon. For more information, read -[Atlassian's documentation](https://support.atlassian.com/jira-cloud-administration/docs/synchronize-jira-cloud-to-bitbucket/). +[Atlassian's documentation](https://support.atlassian.com/jira-cloud-administration/docs/integrate-with-development-tools/). ### `Sync Failed` error when refreshing repository data diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index 1a60ca3a5fe..3293732b59b 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -229,53 +229,60 @@ sudo gitlab-ctl start mattermost ### Mattermost Command Line Tools (CLI) -NOTE: -This CLI will be replaced in a future release with the new [`mmctl` Command Line Tool](https://docs.mattermost.com/manage/mmctl-command-line-tool.html). +[`mmctl`](https://docs.mattermost.com/manage/mmctl-command-line-tool.html) is a CLI tool for the Mattermost server which is installed locally and uses the Mattermost API, but may also be used remotely. You must configure Mattermost either for local connections or authenticate as an administrator with local login credentials (not through GitLab SSO). The executable is located at `/opt/gitlab/embedded/bin/mmctl`. -To use the [Mattermost Command Line Tools (CLI)](https://docs.mattermost.com/administration/command-line-tools.html), ensure that you are in the `/opt/gitlab/embedded/service/mattermost` directory when you run the CLI commands and that you specify the location of the configuration file. The executable is `/opt/gitlab/embedded/bin/mattermost`. +#### Use `mmctl` through a local connection -```shell -cd /opt/gitlab/embedded/service/mattermost +For local connections, the `mmctl` binary and Mattermost must be run from the same server. To enable the local socket: -sudo /opt/gitlab/embedded/bin/chpst -e /opt/gitlab/etc/mattermost/env -P -U mattermost:mattermost -u mattermost:mattermost /opt/gitlab/embedded/bin/mattermost --config=/var/opt/gitlab/mattermost/config.json version -``` +1. Edit `/var/opt/gitlab/mattermost/config.json`, and add the following lines: + + ```json + { + "ServiceSettings": { + ... + "EnableLocalMode": true, + "LocalModeSocketLocation": "/var/tmp/mattermost_local.socket", + ... + } + } + ``` + +1. Restart Mattermost: + + ```shell + sudo gitlab-ctl restart mattermost + ``` -Until [#4745](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4745) has been implemented, the command requires quite of bit typing and is hard to remember, so let's make a bash or Zsh alias to make it a bit easier to remember. Add the following to your `~/.bashrc` or `~/.zshrc` file: +You can then use `/opt/gitlab/embedded/bin/mmctl --local` to run `mmctl` commands +on your Mattermost instance. + +For example, to show the list of users: ```shell -alias mattermost-cli="cd /opt/gitlab/embedded/service/mattermost && sudo /opt/gitlab/embedded/bin/chpst -e /opt/gitlab/etc/mattermost/env -P -U mattermost:mattermost -u mattermost:mattermost /opt/gitlab/embedded/bin/mattermost --config=/var/opt/gitlab/mattermost/config.json $1" +$ /opt/gitlab/embedded/bin/mmctl --local user list + +13dzo5bmg7fu8rdox347hbfxde: appsbot (appsbot@localhost) +tbnkwjdug3dejcoddboo4yuomr: boards (boards@localhost) +wd3g5zpepjgbfjgpdjaas7yj6a: feedbackbot (feedbackbot@localhost) +8d3zzgpurp85zgf1q88pef73eo: playbooks (playbooks@localhost) +There are 4 users on local instance ``` -Then source `~/.zshrc` or `~/.bashrc` with `source ~/.zshrc` or `source ~/.bashrc`. +#### Use `mmctl` through a remote connection -If successful, you can now run any Mattermost CLI command with your new shell alias `mattermost-cli`: +For remote connections or local connections where the socket cannot be used, +create a non SSO user and give that user admin privileges. Those credentials +can then be used to authenticate `mmctl`: ```shell -$ mattermost-cli version - -[sudo] password for username: -{"level":"info","ts":1569614421.9058893,"caller":"utils/i18n.go:83","msg":"Loaded system translations for 'en' from '/opt/gitlab/embedded/service/mattermost/i18n/en.json'"} -{"level":"info","ts":1569614421.9062793,"caller":"app/server_app_adapters.go:58","msg":"Server is initializing..."} -{"level":"info","ts":1569614421.90976,"caller":"sqlstore/supplier.go:223","msg":"Pinging SQL master database"} -{"level":"info","ts":1569614422.0515099,"caller":"mlog/log.go:165","msg":"Starting up plugins"} -{"level":"info","ts":1569614422.0515954,"caller":"app/plugin.go:193","msg":"Syncing plugins from the file store"} -{"level":"info","ts":1569614422.086005,"caller":"app/plugin.go:228","msg":"Found no files in plugins file store"} -{"level":"info","ts":1569614423.9337213,"caller":"sqlstore/post_store.go:1301","msg":"Post.Message supports at most 16383 characters (65535 bytes)"} -{"level":"error","ts":1569614425.6317747,"caller":"go-plugin/stream.go:15","msg":" call to OnConfigurationChange failed, error: Must have a GitLab oauth client id","plugin_id":"com.github.manland.mattermost-plugin-gitlab","source":"plugin_stderr"} -{"level":"info","ts":1569614425.6875598,"caller":"mlog/sugar.go:19","msg":"Ensuring Surveybot exists","plugin_id":"com.mattermost.nps"} -{"level":"info","ts":1569614425.6953356,"caller":"app/server.go:216","msg":"Current version is 5.14.0 (5.14.2/Fri Aug 30 20:20:48 UTC 2019/817ee89711bf26d33f840ce7f59fba14da1ed168/none)"} -{"level":"info","ts":1569614425.6953766,"caller":"app/server.go:217","msg":"Enterprise Enabled: false"} -{"level":"info","ts":1569614425.6954057,"caller":"app/server.go:219","msg":"Current working directory is /opt/gitlab/embedded/service/mattermost/i18n"} -{"level":"info","ts":1569614425.6954265,"caller":"app/server.go:220","msg":"Loaded config","source":"file:///var/opt/gitlab/mattermost/config.json"} -Version: 5.14.0 -Build Number: 5.14.2 -Build Date: Fri Aug 30 20:20:48 UTC 2019 -Build Hash: 817ee89711bf26d33f840ce7f59fba14da1ed168 -Build Enterprise Ready: false -DB Version: 5.14.0 -``` +$ /opt/gitlab/embedded/bin/mmctl auth login http://mattermost.example.com -For more details see [Mattermost Command Line Tools (CLI)](https://docs.mattermost.com/administration/command-line-tools.html) and the [Troubleshooting Mattermost CLI](#troubleshooting-the-mattermost-cli) below. +Connection name: test +Username: local-user +Password: + credentials for "test": "local-user@http://mattermost.example.com" stored +``` ## Configuring GitLab and Mattermost integrations @@ -440,8 +447,7 @@ mattermost['env'] = { } ``` -Refer to the [Mattermost Configuration Settings -documentation](https://docs.mattermost.com/administration/config-settings.html) +Refer to the [Mattermost Configuration Settings documentation](https://docs.mattermost.com/administration/config-settings.html) for details about categories and configuration values. There are a few exceptions to this rule: @@ -512,14 +518,6 @@ sequenceDiagram Mattermost->>User: Mattermost/GitLab user ready ``` -## Troubleshooting the Mattermost CLI - -### Failed to ping DB retrying in 10 seconds err=dial tcp: lookup dockerhost: no such host - -As of version 11.0, majority of the Mattermost settings are now configured via environmental variables. The error is mainly due to the database connection string being commented out in `gitlab.rb` and the database connection settings being set in environmental variables. Additionally, the connection string in the `gitlab.rb` is for MySQL which is no longer supported as of 12.1. - -You can fix this by setting up a `mattermost-cli` [shell alias](#mattermost-command-line-tools-cli). - ## Community support resources For help and support around your GitLab Mattermost deployment please see: diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index aac2820a69e..e297c13a2da 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -7,13 +7,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w # OmniAuth **(FREE SELF)** Users can sign in to GitLab by using their credentials from Twitter, GitHub, and other popular services. -[OmniAuth](https://rubygems.org/gems/omniauth/) is the Rack -framework that GitLab uses to provide this authentication. +[OmniAuth](https://rubygems.org/gems/omniauth/) is the Rack framework that GitLab uses to provide this authentication. - - -If you configure OmniAuth, users can continue to sign in using other -mechanisms, including standard GitLab authentication or LDAP (if configured). +When configured, additional sign-in options are displayed on the sign-in page. ## Supported providers @@ -22,7 +18,6 @@ GitLab supports the following OmniAuth providers. | Provider documentation | OmniAuth provider name | |---------------------------------------------------------------------|----------------------------| | [AliCloud](alicloud.md) | `alicloud` | -| [Atlassian Crowd](../administration/auth/crowd.md) | `crowd` | | [Atlassian](../administration/auth/atlassian.md) | `atlassian_oauth2` | | [Auth0](auth0.md) | `auth0` | | [Authentiq](../administration/auth/authentiq.md) | `authentiq` | @@ -30,7 +25,6 @@ GitLab supports the following OmniAuth providers. | [Azure v2](azure.md) | `azure_activedirectory_v2` | | [Azure v1](azure.md) | `azure_oauth2` | | [Bitbucket Cloud](bitbucket.md) | `bitbucket` | -| [CAS](cas.md) | `cas3` | | [DingTalk](ding_talk.md) | `dingtalk` | | [Facebook](facebook.md) | `facebook` | | [Generic OAuth 2.0](oauth2_generic.md) | `oauth2_generic` | @@ -53,7 +47,7 @@ Setting | Description | Default value ---------------------------|-------------|-------------- `allow_single_sign_on` | Enables you to list the providers that automatically create a GitLab account. The provider names are available in the **OmniAuth provider name** column in the [supported providers table](#supported-providers). | The default is `false`. If `false`, users must be created manually, or they can't sign in using OmniAuth. `auto_link_ldap_user` | If enabled, creates an LDAP identity in GitLab for users that are created through an OmniAuth provider. You can enable this setting if you have the [LDAP (ActiveDirectory)](../administration/auth/ldap/index.md) integration enabled. Requires the `uid` of the user to be the same in both LDAP and the OmniAuth provider. | The default is `false`. -`block_auto_created_users` | If enabled, blocks users that are automatically created from signing in until they are approved by an administrator. | The default is `true`. If you set the value to `false`, make sure you only define providers for `allow_single_sign_on` that you can control, like SAML, Crowd, or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval. +`block_auto_created_users` | If enabled, blocks users that are automatically created from signing in until they are approved by an administrator. | The default is `true`. If you set the value to `false`, make sure you only define providers for `allow_single_sign_on` that you can control, like SAML or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval. To change these settings: @@ -111,6 +105,65 @@ To change these settings: After configuring these settings, you can configure your chosen [provider](#supported-providers). +### Per-provider configuration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89379) in GitLab 15.3. + +If `allow_single_sign_on` is set, GitLab uses one of the following fields returned in the OmniAuth `auth_hash` to establish a username in GitLab for the user signing in, +choosing the first that exists: + +- `username`. +- `nickname`. +- `email`. + +You can create GitLab configuration on a per-provider basis, which is supplied to the [provider](#supported-providers) using `args`. If you set the `gitlab_username_claim` +variable in `args` for a provider, you can select another claim to use for the GitLab username. The chosen claim must be unique to avoid collisions. + +- **For Omnibus installations** + + ```ruby + gitlab_rails['omniauth_providers'] = [ + + # The generic pattern for configuring a provider with name PROVIDER_NAME + + gitlab_rails['omniauth_providers'] = { + name: "PROVIDER_NAME" + ... + args: { gitlab_username_claim: 'sub' } # For users signing in with the provider you configure, the GitLab username will be set to the "sub" received from the provider + }, + + # Here are examples using GitHub and Kerberos + + gitlab_rails['omniauth_providers'] = { + name: "github" + ... + args: { gitlab_username_claim: 'name' } # For users signing in with GitHub, the GitLab username will be set to the "name" received from GitHub + }, + { + name: "kerberos" + ... + args: { gitlab_username_claim: 'uid' } # For users signing in with Kerberos, the GitLab username will be set to the "uid" received from Kerberos + }, + ] + ``` + +- **For installations from source** + + ```yaml + - { name: 'PROVIDER_NAME', + ... + args: { gitlab_username_claim: 'sub' } + } + - { name: 'github', + ... + args: { gitlab_username_claim: 'name' } + } + - { name: 'kerberos', + ... + args: { gitlab_username_claim: 'uid' } + } + ``` + ### Passwords for users created via OmniAuth The [Generated passwords for users created through integrated authentication](../security/passwords_for_integrated_authentication_methods.md) @@ -387,5 +440,4 @@ then override the icon in one of two ways: ## Limitations Most supported OmniAuth providers don't support Git over HTTP password authentication. -The only exception is [Atlassian Crowd](../administration/auth/crowd.md) (since GitLab [13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46935)). As a workaround, you can authenticate using a [personal access token](../user/profile/personal_access_tokens.md). diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 9f707ba9bc6..0c517d07f41 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -709,7 +709,6 @@ args: { security: { authn_requests_signed: true, # enable signature on AuthNRequest want_assertions_signed: true, # enable the requirement of signed assertion - embed_sign: true, # embedded signature or HTTP GET parameter signature metadata_signed: false, # enable signature on Metadata signature_method: 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256', digest_method: 'http://www.w3.org/2001/04/xmlenc#sha256', @@ -756,7 +755,7 @@ Group SAML on a self-managed instance is limited when compared to the recommende [instance-wide SAML](../user/group/saml_sso/index.md). The recommended solution allows you to take advantage of: - [LDAP compatibility](../administration/auth/ldap/index.md). -- [LDAP Group Sync](../user/group/index.md#manage-group-memberships-via-ldap). +- [LDAP Group Sync](../user/group/access_and_permissions.md#manage-group-memberships-via-ldap). - [Required groups](#required-groups). - [Administrator groups](#administrator-groups). - [Auditor groups](#auditor-groups). @@ -801,8 +800,6 @@ If you have any questions on configuring the SAML app, please contact your provi ### Okta setup notes -The following guidance is based on this Okta article, on adding a [SAML Application with an Okta Developer account](https://support.okta.com/help/s/article/Why-can-t-I-add-a-SAML-Application-with-an-Okta-Developer-account?language=en_US): - 1. In the Okta administrator section, make sure to select Classic UI view in the top left corner. From there, choose to **Add an App**. 1. When the app screen comes up you see another button to **Create an App** and choose SAML 2.0 on the next screen. @@ -864,7 +861,7 @@ connect to the Google Workspace SAML app. ### SAML Response -You can find the base64-encoded SAML Response in the [`production_json.log`](../administration/logs.md#production_jsonlog). This response is sent from the IdP, and contains user information that is consumed by GitLab. Many errors in the SAML integration can be solved by decoding this response and comparing it to the SAML settings in the GitLab configuration file. +You can find the base64-encoded SAML Response in the [`production_json.log`](../administration/logs/index.md#production_jsonlog). This response is sent from the IdP, and contains user information that is consumed by GitLab. Many errors in the SAML integration can be solved by decoding this response and comparing it to the SAML settings in the GitLab configuration file. ### GitLab+SAML Testing Environments @@ -907,7 +904,7 @@ the SAML request, but in GitLab 11.7 and earlier this error never reaches GitLab the CSRF check. To bypass this you can add `skip_before_action :verify_authenticity_token` to the -`omniauth_callbacks_controller.rb` file immediately after the `class` line and +`omniauth_callbacks_controller.rb` file immediately before the `after_action :verify_known_sign_in` line and comment out the `protect_from_forgery` line using a `#`. Restart Puma for this change to take effect. This allows the error to hit GitLab, where it can then be seen in the usual logs, or as a flash message on the login screen. @@ -941,8 +938,8 @@ Make sure this information is provided. Another issue that can result in this error is when the correct information is being sent by the IdP, but the attributes don't match the names in the OmniAuth `info` hash. In this case, -you must set `attribute_statements` in the SAML configuration to [map the attribute names in -your SAML Response to the corresponding OmniAuth `info` hash names](#attribute_statements). +you must set `attribute_statements` in the SAML configuration to +[map the attribute names in your SAML Response to the corresponding OmniAuth `info` hash names](#attribute_statements). ### Key validation error, Digest mismatch or Fingerprint mismatch diff --git a/doc/integration/security_partners/index.md b/doc/integration/security_partners/index.md index 50a7b3b717b..507157f9326 100644 --- a/doc/integration/security_partners/index.md +++ b/doc/integration/security_partners/index.md @@ -12,18 +12,18 @@ each security partner: <!-- vale gitlab.Spelling = NO --> -- [Anchore](https://docs.anchore.com/current/docs/using/integration/ci_cd/gitlab/) +- [Anchore](https://docs.anchore.com/current/docs/configuration/integration/ci_cd/gitlab/) - [Bridgecrew](https://docs.bridgecrew.io/docs/integrate-with-gitlab-self-managed) - [Checkmarx](https://checkmarx.atlassian.net/wiki/spaces/SD/pages/1929937052/GitLab+Integration) - [Deepfactor](https://docs.deepfactor.io/hc/en-us/articles/1500008981941) - [GrammaTech](https://www.grammatech.com/codesonar-gitlab-integration) - [Indeni](https://docs.cloudrail.app/#/integrations/gitlab) - [JScrambler](https://docs.jscrambler.com/code-integrity/documentation/gitlab-ci-integration) +- [Mend](https://www.mend.io/gitlab/) - [Semgrep](https://semgrep.dev/for/gitlab) - [StackHawk](https://docs.stackhawk.com/continuous-integration/gitlab.html) - [Tenable](https://docs.tenable.com/tenableio/Content/ContainerSecurity/GetStarted.htm) - [Venafi](https://marketplace.venafi.com/details/gitlab-ci-cd/) - [Veracode](https://community.veracode.com/s/knowledgeitem/gitlab-ci-MCEKSYPRWL35BRTGOVI55SK5RI4A) -- [WhiteSource](https://www.whitesourcesoftware.com/gitlab/) <!-- vale gitlab.Spelling = YES --> diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index f3b66f8c12d..2218529a729 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -12,7 +12,7 @@ Twitter OAuth 2.0 support is [not yet supported](https://gitlab.com/gitlab-org/g To enable the Twitter OmniAuth provider you must register your application with Twitter. Twitter generates a client ID and secret key for you to use. -1. Sign in to [Twitter Application Management](https://apps.twitter.com). +1. Sign in to [Twitter Application Management](https://developer.twitter.com/apps). 1. Select "Create new app". diff --git a/doc/operations/incident_management/img/linked_resources_list_v15_3.png b/doc/operations/incident_management/img/linked_resources_list_v15_3.png Binary files differnew file mode 100644 index 00000000000..2a7b56391a4 --- /dev/null +++ b/doc/operations/incident_management/img/linked_resources_list_v15_3.png diff --git a/doc/operations/incident_management/img/timeline_events_v15_1.png b/doc/operations/incident_management/img/timeline_events_v15_1.png Binary files differindex 3241f35764c..67bc727bc22 100644 --- a/doc/operations/incident_management/img/timeline_events_v15_1.png +++ b/doc/operations/incident_management/img/timeline_events_v15_1.png diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index f1628cb64ca..c1a4c1eb93e 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -13,7 +13,7 @@ tools for the triage, response, and remediation of incidents. Users with at least Guest [permissions](../../user/permissions.md) can access incidents [on public projects](../../user/permissions.md#project-members-permissions). -## Incident Creation +## Incident creation You can create an incident manually or automatically. @@ -205,10 +205,10 @@ field populated. ### Timeline events -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344059) in GitLab 15.2 [with a flag](../../administration/feature_flags.md) named `incident_timeline`. Enabled on GitLab.com. Disabled on self-managed. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344059) in GitLab 15.2 [with a flag](../../administration/feature_flags.md) named `incident_timeline`. Enabled by default. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_timeline`. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `incident_timeline`. On GitLab.com, this feature is available. Incident timelines are an important part of record keeping for incidents. @@ -297,7 +297,7 @@ as a column in the Incidents List, and as a field on newly created Incidents. If the incident isn't closed before the SLA period ends, GitLab adds a `missed::SLA` label to the incident. -## Incident Actions +## Incident actions There are different actions available to help triage and respond to incidents. @@ -376,6 +376,15 @@ for synchronous communication during incident management. After starting a Zoom call for an incident, you can associate the conference call with an issue. Your team members can join the Zoom call without requesting a link. +### Linked resources + +In an incident, you can [links to various resources](linked_resources.md), +for example: + +- The incident Slack channel +- Zoom meeting +- Resources for resolving the incidents + ### Embed metrics in incidents You can embed metrics anywhere [GitLab Markdown](../../user/markdown.md) is diff --git a/doc/operations/incident_management/linked_resources.md b/doc/operations/incident_management/linked_resources.md new file mode 100644 index 00000000000..d2254a30f91 --- /dev/null +++ b/doc/operations/incident_management/linked_resources.md @@ -0,0 +1,66 @@ +--- +stage: Monitor +group: Respond +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Linked resources in incidents **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230852) in GitLab 15.3 [with a flag](../../administration/feature_flags.md) named `incident_resource_links_widget`. Enabled on GitLab.com. Disabled on self-managed. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_resource_links_widget`. +On GitLab.com, this feature is available. + +To help your team members find the important links without having to search through many comments, +you can add linked resources to an incident issue. + +Resources you might want link to: + +- Zoom meetings +- Slack channels or threads +- Google Docs + +## View linked resources of an incident + +Linked resources for an incident are listed under the **Summary** tab. + + + +To view the linked resources of an incident: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Incidents**. +1. Select an incident. + +## Add a linked resource + +Add a linked resource manually from an incident. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To add a linked resource: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Incidents**. +1. Select an incident. +1. In the **Linked resources** section, select the plus icon (**{plus-square}**). +1. Complete the required fields. +1. Select **Add**. + +## Remove a linked resource + +You can also remove a linked resource. + +Prerequisities: + +- You must have at least the Reporter role for the project. + +To remove a linked resource: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Monitor > Incidents**. +1. Select an incident. +1. In the **Linked resources** section, select **Remove** (**{close}**). diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md index 2229ba22be7..17ff0ff01a3 100644 --- a/doc/operations/metrics/embed_grafana.md +++ b/doc/operations/metrics/embed_grafana.md @@ -14,8 +14,8 @@ embed Grafana panels using either: ## Use Grafana-rendered images -You can embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) panels as a -[direct linked rendered image](https://grafana.com/docs/grafana/latest/reference/share_panel/#direct-link-rendered-image). +You can embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) panels as +[a direct link](https://grafana.com/docs/grafana/v7.5/sharing/share-panel/#use-direct-link). Your Grafana instance must: - Be available to the target user, either as a public dashboard or on the same network. @@ -46,7 +46,7 @@ format. To embed panels from a Grafana instance, the data source must be: To set up the Grafana API in Grafana: -1. In Grafana, [generate an Admin-level API Token](https://grafana.com/docs/grafana/latest/http_api/auth/#create-api-token). +1. In Grafana, [generate an Admin-level API Token](https://grafana.com/docs/grafana/next/developers/http_api/auth/#create-api-token). 1. In your GitLab project, go to **Settings > Monitor** and expand the **Grafana authentication** section. 1. To enable the integration, check the **Active** checkbox. diff --git a/doc/policy/alpha-beta-support.md b/doc/policy/alpha-beta-support.md index 45f14e4f9a2..6b0bd66041e 100644 --- a/doc/policy/alpha-beta-support.md +++ b/doc/policy/alpha-beta-support.md @@ -4,7 +4,7 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -<!-- any changes made to this page should be reflected in https://about.gitlab.com/support/statement-of-support.html#alpha--beta-features and https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga --> +<!-- any changes made to this page should be reflected in https://about.gitlab.com/support/statement-of-support/#alpha-beta-features and https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga --> # Support for Alpha, Beta, Limited Availability, and Generally Available Features **(PREMIUM)** diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index e831303988b..e77a0459150 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -16,8 +16,8 @@ Our current policy is: - Backporting security fixes **to the previous two monthly releases in addition to the current stable release**. (See [security releases](#security-releases).) In rare cases, release managers may make an exception and backport to more than -the last two monthly releases. See [Backporting to older -releases](#backporting-to-older-releases) for more information. +the last two monthly releases. See +[Backporting to older releases](#backporting-to-older-releases) for more information. ## Versioning diff --git a/doc/raketasks/backup_gitlab.md b/doc/raketasks/backup_gitlab.md index 8d5ae14a043..4629364ce3d 100644 --- a/doc/raketasks/backup_gitlab.md +++ b/doc/raketasks/backup_gitlab.md @@ -452,8 +452,9 @@ gitlab_rails['backup_upload_storage_options'] = { ##### SSE-KMS -To enable SSE-KMS, you'll need the [KMS key via its Amazon Resource Name (ARN) -in the `arn:aws:kms:region:acct-id:key/key-id` format](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html). Under the `backup_upload_storage_options` configuration setting, set: +To enable SSE-KMS, you'll need the +[KMS key via its Amazon Resource Name (ARN) in the `arn:aws:kms:region:acct-id:key/key-id` format](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html). +Under the `backup_upload_storage_options` configuration setting, set: - `server_side_encryption` to `aws:kms`. - `server_side_encryption_kms_key_id` to the ARN of the key. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 33917ca9410..878511b3e14 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -12,8 +12,8 @@ An application data backup creates an archive file that contains the database, all repositories and all attachments. You can only restore a backup to **exactly the same version and type (CE/EE)** -of GitLab on which it was created. The best way to [migrate your projects -from one server to another](#migrate-to-a-new-server) is through a backup and restore. +of GitLab on which it was created. The best way to +[migrate your projects from one server to another](#migrate-to-a-new-server) is through a backup and restore. WARNING: GitLab doesn't back up items that aren't stored on the file system. If you're @@ -184,14 +184,14 @@ ActiveRecord::StatementInvalid: PG::UndefinedTable ``` Each time the GitLab backup runs, GitLab starts generating 500 errors and errors about missing -tables will [be logged by PostgreSQL](../administration/logs.md#postgresql-logs): +tables will [be logged by PostgreSQL](../administration/logs/index.md#postgresql-logs): ```plaintext ERROR: relation "tablename" does not exist at character 123 ``` -This happens because the task uses `pg_dump`, which [sets a null search -path and explicitly includes the schema in every SQL query](https://gitlab.com/gitlab-org/gitlab/-/issues/23211) +This happens because the task uses `pg_dump`, which +[sets a null search path and explicitly includes the schema in every SQL query](https://gitlab.com/gitlab-org/gitlab/-/issues/23211) to address [CVE-2018-1058](https://www.postgresql.org/about/news/postgresql-103-968-9512-9417-and-9322-released-1834/). Since connections are reused with PgBouncer in transaction pooling mode, @@ -266,7 +266,7 @@ To prepare the new server: [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079) from the old server to avoid man-in-the-middle attack warnings. See [Manually replicate the primary site's SSH host keys](../administration/geo/replication/configuration.md#step-2-manually-replicate-the-primary-sites-ssh-host-keys) for example steps. -1. [Install and configure GitLab](https://about.gitlab.com/install) except +1. [Install and configure GitLab](https://about.gitlab.com/install/) except [incoming email](../administration/incoming_email.md): 1. Install GitLab. 1. Configure by copying `/etc/gitlab` files from the old server to the new server, and update as necessary. @@ -775,7 +775,7 @@ Truncating filenames to resolve the error involves: #### Clean up remote uploaded files -A [known issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/45425) caused object store uploads to remain after a parent resource was deleted. This issue was [resolved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18698). +A [known issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/45425) caused object store uploads to remain after a parent resource was deleted. This issue was [resolved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18698). To fix these files, you must clean up all remote uploaded files that are in the storage but not tracked in the `uploads` database table. diff --git a/doc/raketasks/sidekiq_job_migration.md b/doc/raketasks/sidekiq_job_migration.md index a3bc8b2959a..45a0cbaa267 100644 --- a/doc/raketasks/sidekiq_job_migration.md +++ b/doc/raketasks/sidekiq_job_migration.md @@ -1,40 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../administration/sidekiq/sidekiq_job_migration.md' +remove_date: '2022-11-11' --- -# Sidekiq job migration **(FREE SELF)** +This document was moved to [another location](../administration/sidekiq/sidekiq_job_migration.md). -WARNING: -This operation should be very uncommon. We do not recommend it for the vast majority of GitLab instances. - -Sidekiq routing rules allow administrators to re-route certain background jobs from their regular queue to an alternative queue. By default, GitLab uses one queue per background job type. GitLab has over 400 background job types, and so correspondingly it has over 400 queues. - -Most administrators do not need to change this setting. In some cases with particularly large background job processing workloads, Redis performance may suffer due to the number of queues that GitLab listens to. - -If the Sidekiq routing rules are changed, administrators need to take care with the migration to avoid losing jobs entirely. The basic migration steps are: - -1. Listen to both the old and new queues. -1. Update the routing rules. -1. Wait until there are no publishers dispatching jobs to the old queues. -1. Run the [Rake tasks for future jobs](#future-jobs). -1. Wait for the old queues to be empty. -1. Stop listening to the old queues. - -## Future jobs - -Step 4 involves rewriting some Sidekiq job data for jobs that are already stored in Redis, but due to run in future. There are two sets of jobs to run in future: scheduled jobs and jobs to be retried. We provide a separate Rake task to migrate each set: - -- `gitlab:sidekiq:migrate_jobs:retry` for jobs to be retried. -- `gitlab:sidekiq:migrate_jobs:scheduled` for scheduled jobs. - -Most of the time, running both at the same time is the correct choice. There are two separate tasks to allow for more fine-grained control where needed. To run both at once: - -```shell -# omnibus-gitlab -sudo gitlab-rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule - -# source installations -bundle exec rake gitlab:sidekiq:migrate_jobs:retry gitlab:sidekiq:migrate_jobs:schedule RAILS_ENV=production -``` +<!-- This redirect file can be deleted after <2022-11-11>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/raketasks/spdx.md b/doc/raketasks/spdx.md index 18f058f695e..81da52bc327 100644 --- a/doc/raketasks/spdx.md +++ b/doc/raketasks/spdx.md @@ -19,6 +19,6 @@ sudo gitlab-rake gitlab:spdx:import bundle exec rake gitlab:spdx:import RAILS_ENV=production ``` -To perform this task in the [offline environment](../user/application_security/offline_deployments/#defining-offline-environments), +To perform this task in the [offline environment](../user/application_security/offline_deployments/index.md#defining-offline-environments), an outbound connection to [`licenses.json`](https://spdx.org/licenses/licenses.json) should be allowed. diff --git a/doc/security/crime_vulnerability.md b/doc/security/crime_vulnerability.md index 7956e692887..5a6578f218b 100644 --- a/doc/security/crime_vulnerability.md +++ b/doc/security/crime_vulnerability.md @@ -32,7 +32,7 @@ Although SPDY is enabled in Omnibus installations, CRIME relies on compression ## Nessus -The Nessus scanner, [reports a possible CRIME vulnerability](https://www.tenable.com/plugins/index.php?view=single&id=62565) in GitLab +The Nessus scanner, [reports a possible CRIME vulnerability](https://www.tenable.com/plugins/nessus/62565) in GitLab similar to the following format: ```plaintext @@ -59,7 +59,7 @@ vulnerability. ## References - NGINX ["Module `ngx_http_spdy_module`"](https://nginx.org/en/docs/http/ngx_http_spdy_module.html) -- Tenable Network Security, Inc. ["Transport Layer Security (TLS) Protocol CRIME Vulnerability"](https://www.tenable.com/plugins/index.php?view=single&id=62565) +- Tenable Network Security, Inc. ["Transport Layer Security (TLS) Protocol CRIME Vulnerability"](https://www.tenable.com/plugins/nessus/62565) - Wikipedia contributors, ["CRIME"](https://en.wikipedia.org/wiki/CRIME) Wikipedia, The Free Encyclopedia <!-- ## Troubleshooting diff --git a/doc/security/index.md b/doc/security/index.md index 73ac5028db5..9e05621333b 100644 --- a/doc/security/index.md +++ b/doc/security/index.md @@ -28,8 +28,6 @@ type: index - [Project Import decompressed archive size limits](project_import_decompressed_archive_size_limits.md) - [Responding to security incidents](responding_to_security_incidents.md) -## Securing your GitLab installation +To harden your GitLab instance and minimize the risk of unwanted user account creation, consider access control features like [Sign up restrictions](../user/admin_area/settings/sign_up_restrictions.md) and [Authentication options](../topics/authentication/index.md) . -Consider access control features like [Sign up restrictions](../user/admin_area/settings/sign_up_restrictions.md) and [Authentication options](../topics/authentication/) to harden your GitLab instance and minimize the risk of unwanted user account creation. - -Self-hosting GitLab customers and administrators are responsible for the security of their underlying hosts, and for keeping GitLab itself up to date. It is important to [regularly patch GitLab](../policy/maintenance.md), patch your operating system and its software, and harden your hosts in accordance with vendor guidance. +Self-managed GitLab customers and administrators are responsible for the security of their underlying hosts, and for keeping GitLab itself up to date. It is important to [regularly patch GitLab](../policy/maintenance.md), patch your operating system and its software, and harden your hosts in accordance with vendor guidance. diff --git a/doc/security/information_exclusivity.md b/doc/security/information_exclusivity.md index 0d55881c147..754d5fff843 100644 --- a/doc/security/information_exclusivity.md +++ b/doc/security/information_exclusivity.md @@ -24,8 +24,8 @@ limitation. You can take steps to prevent unintentional sharing and information destruction. This limitation is the reason why only certain people are allowed to [add users to a project](../user/project/members/index.md) -and why only a GitLab administrator can [force push a protected -branch](../user/project/protected_branches.md). +and why only a GitLab administrator can +[force push a protected branch](../user/project/protected_branches.md). <!-- ## Troubleshooting diff --git a/doc/security/reset_user_password.md b/doc/security/reset_user_password.md index 06934b187c1..992a8585a47 100644 --- a/doc/security/reset_user_password.md +++ b/doc/security/reset_user_password.md @@ -81,7 +81,15 @@ If you know the username, user ID, or email address, you can use the Rails conso new_password = ::User.random_password user.password = new_password user.password_confirmation = new_password - ``` + ``` + + To set a specific value for the new password: + + ```ruby + new_password = 'examplepassword' + user.password = new_password + user.password_confirmation = new_password + ``` 1. Optional. Notify the user that an administrator changed their password: diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index 5907860f5cc..93d77a69f0e 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -71,7 +71,7 @@ The following are important notes about 2FA: 2FA for the project. For example, if project *P* belongs to 2FA-enabled group *A* and is shared with 2FA-disabled group *B*, members of group *B* can access project *P* without 2FA. To ensure this scenario doesn't occur, - [prevent sharing of projects](../user/group/index.md#prevent-a-project-from-being-shared-with-groups) + [prevent sharing of projects](../user/group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups) for the 2FA-enabled group. - If you add additional members to a project within a group or subgroup that has 2FA enabled, 2FA is **not** required for those individually added members. @@ -129,6 +129,7 @@ sudo -u git -H bundle exec rake gitlab:two_factor:disable_for_all_users RAILS_EN > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270554) in GitLab 13.7. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/299088) from GitLab Free to GitLab Premium in 13.9. > - It's deployed behind a feature flag, disabled by default. +> - Push notification support [introduced](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/506) in GitLab 15.3. FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `two_factor_for_cli`. On GitLab.com, this feature is not available. The feature is not ready for production use. This feature flag also affects [session duration for Git Operations when 2FA is enabled](../user/admin_area/settings/account_and_limit_settings.md#customize-session-duration-for-git-operations-when-2fa-is-enabled). @@ -136,14 +137,20 @@ On self-managed GitLab, by default this feature is not available. To make it ava Two-factor authentication can be enforced for Git over SSH operations. However, we recommend using [ED25519_SK](../user/ssh.md#ed25519_sk-ssh-keys) or [ECDSA_SK](../user/ssh.md#ecdsa_sk-ssh-keys) SSH keys instead. -The one-time password (OTP) verification can be done using a command: +To perform one-time password (OTP) verification, run: ```shell ssh git@<hostname> 2fa_verify ``` -After the OTP is verified, Git over SSH operations can be used for a session duration of -15 minutes (default) with the associated SSH key. +Then authenticate by either: + +- Entering the correct OTP. +- In GitLab 15.3 and later, responding to a device push notification if + [FortiAuthenticator is enabled](../user/profile/account/two_factor_authentication.md#enable-one-time-password-using-fortiauthenticator). + +After successful authentication, you can perform Git over SSH operations for 15 minutes (default) with the associated +SSH key. ### Security limitation diff --git a/doc/security/user_file_uploads.md b/doc/security/user_file_uploads.md index dcdd18a9f0b..7c11d01396d 100644 --- a/doc/security/user_file_uploads.md +++ b/doc/security/user_file_uploads.md @@ -5,21 +5,45 @@ group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# User File Uploads **(FREE)** +# User file uploads **(FREE)** -Images that are attached to issues, merge requests, or comments -do not require authentication to be viewed if they are accessed directly by URL. -This direct URL contains a random 32-character ID that prevents unauthorized -people from guessing the URL for an image, thus there is some protection if an -image contains sensitive information. +> - Enforced authorization checks [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80117) in GitLab 14.8 [with a flag](../administration/feature_flags.md) named `enforce_auth_checks_on_uploads`. Disabled by default. +> - Enforced authorization checks became [generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352291) in GitLab 15.3. Feature flag `enforce_auth_checks_on_uploads` removed. +> - Project settings in the user interface [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88567) in GitLab 15.3. -Authentication is not enabled because images must be visible in the body of -notification emails, which are often read from email clients that are not -authenticated with GitLab, such as Outlook, Apple Mail, or the Mail app on your -mobile device. +In private or internal projects, GitLab restricts access to uploaded files (such as PDFs) +to authenticated users only. By default, image files are not subject to the same +restriction, and unauthenticated users can use the URL to view the +file. If you enable authorization checks for all media files, images +receive the same protection and are viewable only by authenticated users. -NOTE: -Non-image attachments do require authentication to be viewed. +Users can upload files to issues, merge requests, or comments in a project. Direct URLs +to these images in GitLab contain a random 32-character ID to help prevent +unauthorized users from guessing image URLs. This randomization provides some protection +if an image contains sensitive information. + +Authentication checks for images can cause display issues in the body of notification emails. +Emails are frequently read from clients (such as Outlook, Apple Mail, or your mobile device) +not authenticated with GitLab. Images in emails appear broken and unavailable if +the client is not authorized to GitLab. + +## Enable authorization checks for all media files + +Non-image attachments (including PDFs) always require authentication to be viewed. +You can use this setting to extend this protection to image files. + +Prerequisite: + +- You must have the Maintainer or Owner role for the project. +- Your project visibility settings must be **Private** or **Internal**. + +To configure authentication settings for all media files: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. Scroll to **Project visibility** and select **Require authentication to view media files**. + You cannot select this option for projects with **Public** visibility. <!-- ## Troubleshooting diff --git a/doc/security/webhooks.md b/doc/security/webhooks.md index c3d445103c4..f2066ee4a42 100644 --- a/doc/security/webhooks.md +++ b/doc/security/webhooks.md @@ -74,7 +74,7 @@ allowlist: The allowed entries can be separated by semicolons, commas or whitespaces (including newlines) and be in different formats like hostnames, IP addresses and/or IP ranges. IPv6 is supported. Hostnames that contain Unicode characters should -use [Internationalized Domain Names in Applications](https://www.icann.org/resources/pages/glossary-2014-02-04-en#i) +use [Internationalized Domain Names in Applications](https://www.icann.org/en/icann-acronyms-and-terms/internationalized-domain-names-in-applications-en) (IDNA) encoding. The allowlist can hold a maximum of 1000 entries. Each entry can be a maximum of diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index f936e230a3d..62e045a7593 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -23,11 +23,11 @@ the tiers are no longer mentioned in GitLab documentation: - [Setting a default template for merge requests and issues](../user/project/description_templates.md#set-a-default-template-for-merge-requests-and-issues) - [Email from GitLab](../user/admin_area/email_from_gitlab.md) - Groups: - - [Creating group memberships via CN](../user/group/index.md#create-group-links-via-cn) - - [Group push rules](../user/group/index.md#group-push-rules) - - [Managing group memberships via LDAP](../user/group/index.md#manage-group-memberships-via-ldap) - - [Member locking](../user/group/index.md#prevent-members-from-being-added-to-projects-in-a-group) - - [Overriding user permissions](../user/group/index.md#override-user-permissions) + - [Creating group memberships via CN](../user/group/access_and_permissions.md#create-group-links-via-cn) + - [Group push rules](../user/group/access_and_permissions.md#group-push-rules) + - [Managing group memberships via LDAP](../user/group/access_and_permissions.md#manage-group-memberships-via-ldap) + - [Member locking](../user/group/access_and_permissions.md#prevent-members-from-being-added-to-projects-in-a-group) + - [Overriding user permissions](../user/group/access_and_permissions.md#override-user-permissions) - [User contribution analytics](../user/group/contribution_analytics/index.md) - [Kerberos integration](../integration/kerberos.md) - Issue boards: @@ -60,8 +60,8 @@ the tiers are no longer mentioned in GitLab documentation: - Rake tasks for [LDAP tasks](../administration/raketasks/ldap.md), including [syncing groups](../administration/raketasks/ldap.md#run-a-group-sync) - Logging: - - [`audit_json.log`](../administration/logs.md#audit_jsonlog) (specific entries) - - [`elasticsearch.log`](../administration/logs.md#elasticsearchlog) + - [`audit_json.log`](../administration/logs/index.md#audit_jsonlog) (specific entries) + - [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) - Merge requests: - [Full code quality reports in the code quality tab](../ci/testing/code_quality.md#code-quality-reports) - [Merge request approvals](../user/project/merge_requests/approvals/index.md) diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index da84cc6211e..6b83a00cac1 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -33,10 +33,10 @@ To subscribe to GitLab SaaS: and decide which tier you want. 1. Create a user account for yourself by using the [sign up page](https://gitlab.com/users/sign_up). -1. Create a [group](../../user/group/index.md#create-a-group). Your license tier applies to the top-level group, its subgroups, and projects. +1. Create a [group](../../user/group/manage.md#create-a-group). Your subscription tier applies to the top-level group, its subgroups, and projects. 1. Create additional users and - [add them to the group](../../user/group/index.md#add-users-to-a-group). The users in this group, its subgroups, and projects can use - the features of your license tier, and they consume a seat in your subscription. + [add them to the group](../../user/group/manage.md#add-users-to-a-group). The users in this group, its subgroups, and projects can use + the features of your subscription tier, and they consume a seat in your subscription. 1. On the left sidebar, select **Billing** and choose a tier. 1. Fill out the form to complete your purchase. @@ -73,7 +73,7 @@ subscription according to the maximum number of users assigned to the top-level add and remove users during the subscription period, as long as the total users at any given time doesn't exceed the subscription count. -A top-level group can be [changed](../../user/group/index.md#change-a-groups-path) like any other group. +A top-level group can be [changed](../../user/group/manage.md#change-a-groups-path) like any other group. Every user is included in seat usage, with the following exceptions: @@ -185,7 +185,7 @@ To remove a billable user from your subscription: 1. In the row for the user you want to remove, on the right side, select the ellipsis and **Remove user**. 1. Re-type the username and select **Remove user**. -If you add a member to a group by using the [share a group with another group](../../user/group/index.md#share-a-group-with-another-group) feature, you can't remove the member by using this method. Instead, you can either: +If you add a member to a group by using the [share a group with another group](../../user/group/manage.md#share-a-group-with-another-group) feature, you can't remove the member by using this method. Instead, you can either: - Remove the member from the shared group. You must be a group owner to do this. - From the group's membership page, remove access from the entire shared group. @@ -297,10 +297,10 @@ for your personal or group namespace. CI/CD minutes are a **one-time purchase**, ## Add-on subscription for additional Storage and Transfer NOTE: -Free namespaces are subject to a 5GB storage and 10GB transfer [soft limit](https://about.gitlab.com/pricing). Once all storage is available to view in the usage quota workflow, GitLab will automatically enforce the namespace storage limit and the project limit will be removed. This change will be announced separately. The storage and transfer add-on can be purchased to increase the limits. +Free namespaces are subject to a 5GB storage and 10GB transfer [soft limit](https://about.gitlab.com/pricing/). Once all storage is available to view in the usage quota workflow, GitLab will automatically enforce the namespace storage limit and the project limit will be removed. This change will be announced separately. The storage and transfer add-on can be purchased to increase the limits. -Projects have a free storage quota of 10 GB. To exceed this quota you must first [purchase one or -more storage subscription units](#purchase-more-storage-and-transfer). Each unit provides 10 GB of additional +Projects have a free storage quota of 10 GB. To exceed this quota you must first +[purchase one or more storage subscription units](#purchase-more-storage-and-transfer). Each unit provides 10 GB of additional storage per namespace. A storage subscription is renewed annually. For more details, see [Usage Quotas](../../user/usage_quotas.md). diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 4caf69d6015..ed96fbd91ef 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -109,8 +109,8 @@ Purchases in the Customers Portal require a credit card on record as a payment m multiple credit cards to your account, so that purchases for different products are charged to the correct card. -If you would like to use an alternative method to pay, please [contact our Sales -team](https://about.gitlab.com/sales/). +If you would like to use an alternative method to pay, please +[contact our Sales team](https://about.gitlab.com/sales/). To change your payment method: diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index c1a11ce7d12..758b472d67b 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -255,7 +255,7 @@ It also displays the following information: | Field | Description | |:-------------------|:------------| | Users in License | The number of users you've paid for in the current license loaded on the system. The number does not change unless you [add seats](#add-seats-to-a-subscription) during your current subscription period. | -| Billable users | The daily count of billable users on your system. The count may change as you block or add users to your instance. | +| Billable users | The daily count of billable users on your system. The count may change as you block, deactivate, or add users to your instance. | | Maximum users | The highest number of billable users on your system during the term of the loaded license. | | Users over license | Calculated as `Maximum users` - `Users in License` for the current license term. This number incurs a retroactive charge that must be paid before renewal. | @@ -274,18 +274,18 @@ This file contains the information GitLab uses to manually process quarterly rec The **License Usage** CSV includes the following details: - License key -- Email +- Licensee email - License start date - License end date - Company - Generated at (the timestamp for when the file was exported) - Table of historical user counts for each day in the period: - - Date the count was recorded - - Active user count + - Timestamp the count was recorded + - Billable user count NOTES: -- All timestamps are displayed in UTC. +- All date timestamps are displayed in UTC. - A custom format is used for [dates](https://gitlab.com/gitlab-org/gitlab/blob/3be39f19ac3412c089be28553e6f91b681e5d739/config/initializers/date_time_formats.rb#L7) and [times](https://gitlab.com/gitlab-org/gitlab/blob/3be39f19ac3412c089be28553e6f91b681e5d739/config/initializers/date_time_formats.rb#L13) in CSV files. ## Renew your subscription @@ -312,7 +312,7 @@ the contact person who manages your subscription. It's important to regularly review your user accounts, because: -- Stale user accounts that are not blocked count as billable users. You may pay more than you should +- Stale user accounts may count as billable users. You may pay more than you should if you renew for too many users. - Stale user accounts can be a security risk. A regular review helps reduce this risk. @@ -329,7 +329,7 @@ To view the number of _users over license_ go to the **Admin Area**. You purchase a license for 10 users. -| Event | Billable members | Maximum users | +| Event | Billable users | Maximum users | |:---------------------------------------------------|:-----------------|:--------------| | Ten users occupy all 10 seats. | 10 | 10 | | Two new users join. | 12 | 12 | diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md index b3ce2aa1683..1fea573faa1 100644 --- a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md @@ -35,7 +35,7 @@ you need a [Google Cloud Platform account](https://console.cloud.google.com). Sign in with an existing Google account, such as the one you use to access Gmail or Google Drive, or create a new one. -1. Follow the steps described in the ["Before you begin" section](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin) +1. Follow the steps described in the ["Before you begin" section](https://cloud.google.com/kubernetes-engine/docs/deploy-app-cluster#before-you-begin) of the Kubernetes Engine documentation to enable the required APIs and related services. 1. Ensure you've created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account) with Google Cloud Platform. diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index d8734ab5b13..7f9707a6939 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -460,7 +460,7 @@ The following table lists variables used to disable jobs. | **Job Name** | **CI/CDVariable** | **GitLab version** | **Description** | |----------------------------------------|---------------------------------|-----------------------|-----------------| -| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/) about how `.fuzz_base` provide capability for your own jobs. If the variable is present, your jobs aren't created. | +| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/index.md) about how `.fuzz_base` provide capability for your own jobs. If the variable is present, your jobs aren't created. | | `apifuzzer_fuzz` | `API_FUZZING_DISABLED` | [From GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39135) | If the variable is present, the job isn't created. | | `build` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | | `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 6d4f6eb698e..dc58f42f30e 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -267,7 +267,10 @@ The GitLab integration with Helm does not support installing applications when behind a proxy. To do so, inject proxy settings into the installation pods at runtime. -For example, you can use a [`PodPreset`](https://v1-19.docs.kubernetes.io/docs/concepts/workloads/pods/podpreset/): +For example, you can use a `PodPreset`: + +NOTE: +[PodPreset was removed in Kubernetes v1.20](https://github.com/kubernetes/kubernetes/pull/94090). ```yaml apiVersion: settings.k8s.io/v1alpha1 diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 8d35fd245d5..4e8d0e08eff 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -199,7 +199,7 @@ see the documentation. ## Auto Secret Detection > - Introduced in GitLab 13.1. -> - Select functionality [made available](../../user/application_security/secret_detection/#making-secret-detection-available-to-all-gitlab-tiers) in all tiers in GitLab 13.3 +> - Select functionality [made available](../../user/application_security/secret_detection/index.md#making-secret-detection-available-to-all-gitlab-tiers) in all tiers in GitLab 13.3 Secret Detection uses the [Secret Detection Docker image](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) to run Secret Detection on the current code, and checks for leaked secrets. Auto Secret Detection requires [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or above. @@ -458,8 +458,8 @@ To use Auto Deploy on a Kubernetes 1.16+ cluster: ``` 1. If you have an in-cluster PostgreSQL database installed with - `AUTO_DEVOPS_POSTGRES_CHANNEL` set to `1`, follow the [guide to upgrade - PostgreSQL](upgrading_postgresql.md). + `AUTO_DEVOPS_POSTGRES_CHANNEL` set to `1`, follow the + [guide to upgrade PostgreSQL](upgrading_postgresql.md). 1. If you are deploying your application for the first time and are using GitLab 12.9 or 12.10, set `AUTO_DEVOPS_POSTGRES_CHANNEL` to `2`. diff --git a/doc/topics/autodevops/troubleshooting.md b/doc/topics/autodevops/troubleshooting.md index cee4ba68b49..045f843be44 100644 --- a/doc/topics/autodevops/troubleshooting.md +++ b/doc/topics/autodevops/troubleshooting.md @@ -100,7 +100,7 @@ WARNING: Setting `POSTGRES_ENABLED` to `false` permanently deletes any existing channel 1 database for your environment. -## Error: unable to recognize "": no matches for kind "Deployment" in version "extensions/v1beta1" +## `Error: unable to recognize "": no matches for kind "Deployment" in version "extensions/v1beta1"` After upgrading your Kubernetes cluster to [v1.16+](stages.md#kubernetes-116), you may encounter this message when deploying with Auto DevOps: @@ -148,7 +148,7 @@ that works for this problem. Follow these steps to use the tool in Auto DevOps: 1. Continue the deployments as usual. -## Error: error initializing: Looks like "https://kubernetes-charts.storage.googleapis.com" is not a valid chart repository or cannot be reached +## `Error: error initializing: Looks like "https://kubernetes-charts.storage.googleapis.com" is not a valid chart repository or cannot be reached` As [announced in the official CNCF blog post](https://www.cncf.io/blog/2020/10/07/important-reminder-for-all-helm-users-stable-incubator-repos-are-deprecated-and-all-images-are-changing-location/), the stable Helm chart repository was deprecated and removed on November 13th, 2020. @@ -197,7 +197,7 @@ To fix your custom chart: You can find more information in [issue #263778, "Migrate PostgreSQL from stable Helm repository"](https://gitlab.com/gitlab-org/gitlab/-/issues/263778). -## Error: release .... failed: timed out waiting for the condition +## `Error: release .... failed: timed out waiting for the condition` When getting started with Auto DevOps, you may encounter this error when first deploying your application: diff --git a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md index b0814ac5076..5772a891b41 100644 --- a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md +++ b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md @@ -68,8 +68,7 @@ The v1 chart is backward compatible with the v0 chart, so no configuration chang ### Upgrade deployments to the v2 `auto-deploy-image` The v2 auto-deploy-image contains multiple dependency and architectural changes. -If your Auto DevOps project has an active environment deployed with the v1 `auto-deploy-image`, -please proceed with the following upgrade guide. Otherwise, you can skip this process. +If your Auto DevOps project has an active environment deployed with the v1 `auto-deploy-image`, proceed with the following upgrade guide. Otherwise, you can skip this process. #### Kubernetes 1.16+ @@ -276,4 +275,4 @@ you might encounter the following error: - `Error: rendered manifests contain a resource that already exists. Unable to continue with install: Secret "production-postgresql" in namespace "<project-name>-production" exists and cannot be imported into the current release: invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"; annotation validation error: missing key "meta.helm.sh/release-name": must be set to "production-postgresql"; annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "<project-name>-production"` This is because the previous deployment was deployed with Helm2, which is not compatible with Helm3. -To resolve the problem, please follow the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). +To resolve the problem, follow the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md index 258195bb89f..0d8d8cd6579 100644 --- a/doc/topics/autodevops/upgrading_postgresql.md +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -131,8 +131,7 @@ being modified after the database dump is created. ## Retain persistent volumes -By default the [persistent -volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +By default the [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) used to store the underlying data for PostgreSQL is marked as `Delete` when the pods and pod claims that use the volume is deleted. diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index d40690d86c6..6fd7f4a5207 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -50,7 +50,8 @@ repository with Git LFS. For example, if you want to upload a very large file an check it into your Git repository: ```shell -git clone git@gitlab.example.com:group/project.git +git clone git@gitlab.example.com:group/my-sample-project.git +cd my-sample-project git lfs install # initialize the Git LFS project git lfs track "*.iso" # select the file extensions that you want to treat as large files ``` @@ -70,6 +71,8 @@ LFS doesn't work properly for people cloning the project: ```shell git add .gitattributes +git commit -am "Added .gitattributes to capture LFS tracking" +git push origin main ``` Cloning the repository works the same as before. Git automatically detects the @@ -78,7 +81,7 @@ command with a SSH URL, you have to enter your GitLab credentials for HTTP authentication. ```shell -git clone git@gitlab.example.com:group/project.git +git clone git@gitlab.example.com:group/my-sample-project.git ``` If you already cloned the repository and you want to get the latest LFS object @@ -93,7 +96,7 @@ and are not pushed to the remote repository. ### Migrate an existing repository to Git LFS -Read the documentation on how to [migrate an existing Git repository with Git LFS](https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-migrate.1.ronn). +Read the documentation on how to [migrate an existing Git repository with Git LFS](https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-migrate.adoc). ### Removing objects from LFS @@ -112,8 +115,7 @@ See the documentation on [File Locking](../../../user/project/file_lock.md). > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46572) in GitLab 13.6. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62539) in GitLab 14.0. Feature flag `include_lfs_blobs_in_archive` removed. -Prior to GitLab 13.5, [project source -downloads](../../../user/project/repository/index.md) would include Git +Prior to GitLab 13.5, [project source downloads](../../../user/project/repository/index.md) would include Git LFS pointers instead of the actual objects. For example, LFS pointers look like the following: @@ -214,7 +216,7 @@ This behavior is caused by Git LFS using HTTPS connections by default when a To prevent this from happening, set the LFS URL in project Git configuration: ```shell -git config --add lfs.url "http://gitlab.example.com/group/project.git/info/lfs" +git config --add lfs.url "http://gitlab.example.com/group/my-sample-project.git/info/lfs" ``` ### Credentials are always required when pushing an object diff --git a/doc/topics/git/terminology.md b/doc/topics/git/terminology.md index 4ce87aa2d11..f64e614c253 100644 --- a/doc/topics/git/terminology.md +++ b/doc/topics/git/terminology.md @@ -31,7 +31,7 @@ This copy is called a [**fork**](../../user/project/repository/forking_workflow. The process is called "creating a fork." When you fork a repository, you create a copy of the project in your own -[namespace](../../user/group/#namespaces). You then have write permissions to modify the project files +[namespace](../../user/namespace/index.md). You then have write permissions to modify the project files and settings. For example, you can fork this project, <https://gitlab.com/gitlab-tests/sample-project/>, into your namespace. diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index 13962ad0376..36c26a02064 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -212,10 +212,10 @@ apply more than one: [`gitlab.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/13.5.1+ee.0/files/gitlab-config-template/gitlab.rb.template#L1435-1455) file: ```shell - omnibus_gitconfig['system'] = { + gitaly['gitconfig'] = [ # Set the http.postBuffer size, in bytes - "http" => ["postBuffer = 524288000"] - } + {key: "http.postBuffer", value: "524288000"}, + ] ``` 1. After applying this change, apply the configuration change: diff --git a/doc/topics/offline/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index aa923eb6dc6..353ba094d1e 100644 --- a/doc/topics/offline/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -139,8 +139,7 @@ sudo cp /etc/gitlab/ssl/my-host.internal.crt /etc/gitlab-runner/certs/ca.crt ## Enabling GitLab Runner -[Following a similar process to the steps for installing our GitLab Runner as a -Docker service](https://docs.gitlab.com/runner/install/docker.html#docker-image-installation), we must first register our runner: +[Following a similar process to the steps for installing our GitLab Runner as a Docker service](https://docs.gitlab.com/runner/install/docker.html#install-the-docker-image-and-start-the-container), we must first register our runner: ```shell $ sudo docker run --rm -it -v /etc/gitlab-runner:/etc/gitlab-runner gitlab/gitlab-runner register diff --git a/doc/topics/release_your_application.md b/doc/topics/release_your_application.md index 6c94e9e78f9..61ca1468dca 100644 --- a/doc/topics/release_your_application.md +++ b/doc/topics/release_your_application.md @@ -30,14 +30,14 @@ to Kubernetes clusters using the [GitLab agent](../user/clusters/agent/install/i #### GitOps deployments **(PREMIUM)** -With the [GitLab agent for Kubernetes](../user/clusters/agent/install/index.md), you can perform [pull-based -deployments of Kubernetes manifests](../user/clusters/agent/gitops.md). This provides a scalable, secure, and cloud-native -approach to manage Kubernetes deployments. +With the [GitLab agent for Kubernetes](../user/clusters/agent/install/index.md), you can perform +[pull-based deployments of Kubernetes manifests](../user/clusters/agent/gitops.md). This provides a scalable, secure, +and cloud-native approach to manage Kubernetes deployments. #### Deploy to Kubernetes from GitLab CI/CD -With the [GitLab agent for Kubernetes](../user/clusters/agent/install/index.md), you can perform [push-based -deployments](../user/clusters/agent/ci_cd_workflow.md) from GitLab CI/CD. The agent provides +With the [GitLab agent for Kubernetes](../user/clusters/agent/install/index.md), you can perform +[push-based deployments](../user/clusters/agent/ci_cd_workflow.md) from GitLab CI/CD. The agent provides a secure and reliable connection between GitLab and your Kubernetes cluster. ### Deploy to AWS with GitLab CI/CD diff --git a/doc/topics/set_up_organization.md b/doc/topics/set_up_organization.md index c8183a0757e..b6cacf0b14e 100644 --- a/doc/topics/set_up_organization.md +++ b/doc/topics/set_up_organization.md @@ -9,6 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Configure your organization and its users. Determine user roles and give everyone access to the projects they need. +- [Namespaces](../user/namespace/index.md) - [Members](../user/project/members/index.md) - [Workspace](../user/workspace/index.md) _(In development)_ - [Groups](../user/group/index.md) diff --git a/doc/tutorials/make_your_first_git_commit.md b/doc/tutorials/make_your_first_git_commit.md index a35137e158b..be9023c6ae0 100644 --- a/doc/tutorials/make_your_first_git_commit.md +++ b/doc/tutorials/make_your_first_git_commit.md @@ -37,7 +37,7 @@ Each time you push a change, Git records it as a unique *commit*. These commits the history of when and how a file changed, and who changed it. ```mermaid -graph LR +graph TB subgraph Repository commit history A(Author: Alex<br>Date: 3 Jan at 1PM<br>Commit message: Added sales figures for January<br> Commit ID: 123abc12) ---> B B(Author: Sam<br>Date: 4 Jan at 10AM<br>Commit message: Removed outdated marketing information<br> Commit ID: aabb1122) ---> C @@ -54,7 +54,7 @@ of a repository are in a default branch. To make changes, you: 1. When you're ready, *merge* your branch into the default branch. ```mermaid -flowchart LR +flowchart TB subgraph Default branch A[Commit] --> B[Commit] --> C[Commit] --> D[Commit] end diff --git a/doc/tutorials/move_personal_project_to_a_group.md b/doc/tutorials/move_personal_project_to_a_group.md index fdda42be3fa..5ebbf813ab9 100644 --- a/doc/tutorials/move_personal_project_to_a_group.md +++ b/doc/tutorials/move_personal_project_to_a_group.md @@ -24,7 +24,7 @@ A group gives you some great benefits. For example, you can: However, if you're working in a [personal project](../user/project/working_with_projects.md#view-personal-projects), you can't use these features. Personal projects are created under your -[personal namespace](../user/group/index.md#namespaces). They're not part of a group, +[personal namespace](../user/namespace/index.md). They're not part of a group, so you can't get any of the benefits and features of a group. But don't worry! You can move your existing personal project to a group. diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index 81b98b95068..b69f8de2947 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -45,6 +45,98 @@ sole discretion of GitLab Inc. <div class="announcement-milestone"> +## Announced in 15.8 + +<div class="deprecation removal-160 breaking-change"> + +### Security report schemas version 14.x.x + +End of Support: GitLab <span class="removal-milestone">15.8</span> (2023-01-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +All [security report schema](https://gitlab.com/gitlab-org/security-products/security-report-schemas) versions before 15.0.0 are considered deprecated in GitLab %15.8. Specifically, all [schemas](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/lib/ee/gitlab/ci/parsers/security/validators/schemas) that match 14.*.* will be deprecated. + +Please note that any [security report scanner integration](https://docs.gitlab.com/ee/development/integrations/secure.html) with GitLab using a deprecated schema version will result in a deprecation warning as a result of [report validation](https://docs.gitlab.com/ee/development/integrations/secure.html#report-validation). + +See [Security report validation](https://docs.gitlab.com/ee/user/application_security/#security-report-validation) for more information. + +</div> +</div> + +<div class="announcement-milestone"> + +## Announced in 15.3 + +<div class="deprecation removal-160 breaking-change"> + +### Atlassian Crowd OmniAuth provider + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `omniauth_crowd` gem that provides GitLab with the Atlassian Crowd OmniAuth provider will be removed in our +next major release, GitLab 16.0. This gem sees very little use and its +[lack of compatibility](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is +[blocking our upgrade](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### CAS OmniAuth provider + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider will be removed in our next major +release, GitLab 16.0. This gem sees very little use and its lack of upstream maintenance is preventing GitLab's +[upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). + +</div> + +<div class="deprecation removal-160"> + +### Maximum number of active pipelines per project limit (`ci_active_pipelines`) + +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the Rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection: + +- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html). +- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits). + +</div> + +<div class="deprecation removal-160 breaking-change"> + +### Redis 5 deprecated + +End of Support: GitLab <span class="removal-milestone">15.6</span> (2022-11-22) +Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With GitLab 13.9, in the Omnibus GitLab package and GitLab Helm chart 4.9, the Redis version [was updated to Redis 6](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#omnibus-improvements). +Redis 5 has reached the end of life in April 2022 and will no longer be supported as of GitLab 15.6. +If you are using your own Redis 5.0 instance, you should upgrade it to Redis 6.0 or higher before upgrading to GitLab 16.0 or higher. + +</div> +</div> + +<div class="announcement-milestone"> + ## Announced in 15.2 <div class="deprecation removal-160 breaking-change"> @@ -54,7 +146,7 @@ sole discretion of GitLab Inc. Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `job_age` parameter, returned from the `POST /jobs/request` API endpoint used in communication with GitLab Runner, was never used by any GitLab or Runner feature. This parameter will be removed in GitLab 16.0. @@ -75,7 +167,7 @@ This could be a breaking change for anyone that developed their own runner that Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [Jira DVCS Connector](https://docs.gitlab.com/ee/integration/jira/dvcs.html) (which enables the [Jira Development Panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/)), will no longer support Jira Cloud users starting with GitLab 16.0. The [GitLab for Jira App](https://docs.gitlab.com/ee/integration/jira/connect-app.html) has always been recommended for Jira Cloud users, and it will be required instead of the DVCS connector. If you are a Jira Cloud user, we recommended you begin migrating to the GitLab for Jira App. @@ -90,7 +182,7 @@ Any Jira Server and Jira Data Center users will need to confirm they are not usi Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https://gitlab.com/gitlab-org/gitlab/-/issues/335372) to include a new `title` field. This field is an alias for the current `name` field, making the less specific `name` field redundant. The `name` field will be removed from the PipelineSecurityReportFinding type in GitLab 16.0. @@ -104,7 +196,7 @@ Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https:/ Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [`project_fingerprint`](https://gitlab.com/groups/gitlab-org/-/epics/2791) attribute of vulnerability findings is being deprecated in favor of a `uuid` attribute. By using UUIDv5 values to identify findings, we can easily associate any related entity with a finding. The `project_fingerprint` attribute is no longer being used to track findings, and will be removed in GitLab 16.0. @@ -118,7 +210,7 @@ The [`project_fingerprint`](https://gitlab.com/groups/gitlab-org/-/epics/2791) a Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `maintainer_note` argument in the `POST /runners` REST endpoint was deprecated in GitLab 14.8 and replaced with the `maintenance_note` argument. @@ -146,7 +238,7 @@ GitLab 15.3 to simplify the codebase and prevent any unwanted performance degrad Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Previous work helped [align the vulnerabilities calls for pipeline security tabs](https://gitlab.com/gitlab-org/gitlab/-/issues/343469) to match the vulnerabilities calls for project-level and group-level vulnerability reports. This helped the frontend have a more consistent interface. The old `project.pipeline.securityReportFindings` query was formatted differently than other vulnerability data calls. Now that it has been replaced with the new `project.pipeline.vulnerabilities` field, the old `project.pipeline.securityReportFindings` is being deprecated and will be removed in GitLab 16.0. @@ -165,7 +257,7 @@ Previous work helped [align the vulnerabilities calls for pipeline security tabs Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `CiCdSettingsUpdate` mutation was renamed to `ProjectCiCdSettingsUpdate` in GitLab 15.0. @@ -182,7 +274,7 @@ instead. Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `legacyMode` argument to the `status` field in `RunnerType` will be rendered non-functional in the 16.0 release @@ -201,7 +293,7 @@ be present during the 16.x cycle to avoid breaking the API signature, and will b Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Support for PostgreSQL 12 is scheduled for removal in GitLab 16.0. @@ -214,16 +306,15 @@ Upgrading to PostgreSQL 13 is not yet supported for GitLab instances with Geo en </div> -<div class="deprecation removal-152"> +<div class="deprecation removal-153"> ### Vulnerability Report sort by State -Planned removal: GitLab <span class="removal-milestone">15.2</span> (2022-07-22) +Planned removal: GitLab <span class="removal-milestone">15.3</span> (2022-08-22) The ability to sort the Vulnerability Report by the `State` column was disabled and put behind a feature flag in GitLab 14.10 due to a refactor of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting -by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed in -GitLab 15.2 to simplify the codebase and prevent any unwanted performance degradation. +by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed to simplify the codebase and prevent any unwanted performance degradation. </div> </div> @@ -239,7 +330,7 @@ GitLab 15.2 to simplify the codebase and prevent any unwanted performance degrad Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 15.0, for Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_Java_Version` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). @@ -253,7 +344,7 @@ In GitLab 15.0, for Dependency Scanning, the default version of Java that the sc Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Manual iteration management is deprecated and only automatic iteration cadences will be supported in the future. @@ -285,7 +376,7 @@ For more information about iteration cadences, you can refer to Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As Advanced Search migrations usually require support multiple code paths for a long period of time, it’s important to clean those up when we safely can. We use GitLab major version upgrades as a safe time to remove backward compatibility for indices that have not been fully migrated. See the [upgrade documentation](https://docs.gitlab.com/ee/update/index.html#upgrading-to-a-new-major-version) for details. @@ -299,7 +390,7 @@ As Advanced Search migrations usually require support multiple code paths for a Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. Updating notes confidential attribute is no longer supported by any means. We are changing this to simplify the experience and prevent private information from being unintentionally exposed. @@ -318,7 +409,7 @@ Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. U Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` to upload files is deprecated and will be fully removed in GitLab 15.0. Review the [15.0 specific changes](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html) for the [removed background uploads settings for object storage](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#removed-background-uploads-settings-for-object-storage). @@ -359,7 +450,7 @@ In 15.0, support for daemon mode for GitLab Pages will be removed. Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab self-monitoring gives administrators of self-hosted GitLab instances the tools to monitor the health of their instances. This feature is deprecated in GitLab 14.9, and is scheduled for removal in 16.0. @@ -373,7 +464,7 @@ GitLab self-monitoring gives administrators of self-hosted GitLab instances the Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. @@ -404,7 +495,7 @@ The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks. Planned removal: GitLab <span class="removal-milestone">14.10</span> (2022-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. @@ -420,7 +511,7 @@ Downloading Composer dependencies without authentication is deprecated in GitLab Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. @@ -436,7 +527,7 @@ Since it isn't used in the context of GitLab (the product), `htpasswd` authentic Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `user_email_lookup_limit` [API field](https://docs.gitlab.com/ee/api/settings.html) is deprecated and will be removed in GitLab 15.0. Until GitLab 15.0, `user_email_lookup_limit` is aliased to `search_rate_limit` and existing workflows will continue to work. @@ -470,7 +561,7 @@ This change is part of regular maintenance to keep our codebase clean. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). @@ -493,7 +584,7 @@ For additional context, or to provide feedback regarding this change, please ref Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. For those using Dependency Scanning for Python projects, we are deprecating the default `gemnasium-python:2` image which uses Python 3.6 as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). @@ -564,7 +655,7 @@ The following `geo:db:*` tasks will be replaced with their corresponding `db:*:g Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede CODEOWNERS. The CODEOWNERS feature will no longer be available for access control. @@ -578,7 +669,7 @@ The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 1 Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). @@ -596,7 +687,7 @@ GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configu Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Elasticsearch 6.8 is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. @@ -614,7 +705,7 @@ Elasticsearch 6.8 is also incompatible with Amazon OpenSearch, which we [plan to Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to @@ -642,7 +733,7 @@ To align with this change, API calls to list external status checks will also re Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We are removing a non-standard extension to our GraphQL processor, which we added for backwards compatibility. This extension modifies the validation of GraphQL queries, allowing the use of the `ID` type for arguments where it would normally be rejected. @@ -706,7 +797,7 @@ an inline argument expression). Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens @@ -728,7 +819,7 @@ tokens before GitLab 15.0 is released: Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The feature to disable enforcement of PAT expiration is unusual from a security perspective. @@ -744,7 +835,7 @@ Unexpected behavior in a security feature is inherently dangerous, so we have de Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The feature to disable enforcement of SSH expiration is unusual from a security perspective. @@ -760,7 +851,7 @@ Unexpected behavior in a security feature is inherently dangerous, so we have de Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [GitLab SAST SpotBugs analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) scans [Java, Scala, Groovy, and Kotlin code](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) for security vulnerabilities. @@ -785,7 +876,7 @@ If you rely on Java 8 being present in the analyzer environment, you must take a Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTrendsMeasurements` in 13.10 and the old field name has been marked as deprecated. To fix the existing GraphQL queries, replace `instanceStatisticsMeasurements` with `usageTrendsMeasurements`. @@ -799,7 +890,7 @@ The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTren Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Runner REST endpoints will stop accepting `paused` or `active` as a status value in GitLab 16.0. @@ -819,7 +910,7 @@ When checking for active runners, specify `paused=false`. Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `GET /groups/:id/runners?type=project_type` endpoint will be removed in GitLab 16.0. The endpoint always returned an empty collection. @@ -833,7 +924,7 @@ The `GET /groups/:id/runners?type=project_type` endpoint will be removed in GitL Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Occurrences of the `active` identifier in the GitLab Runner REST and GraphQL API endpoints will be @@ -865,7 +956,7 @@ existing runners. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. [Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/request_profiling.html) is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. @@ -885,7 +976,7 @@ For more information, check the [summary section of the deprecation issue](https Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) feature is deprecated in GitLab 14.8 for Premium customers and is scheduled for removal in GitLab 15.0. This feature is not deprecated for GitLab Ultimate customers. @@ -904,7 +995,7 @@ This change will also help GitLab remain consistent in its tiering strategy with Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As of 14.8 the retire.js job is being deprecated from Dependency Scanning. It will continue to be included in our CI/CD template while deprecated. We are removing retire.js from Dependency Scanning on May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. @@ -920,7 +1011,7 @@ If you have explicitly excluded retire.js using DS_EXCLUDED_ANALYZERS you will n Planned removal: GitLab <span class="removal-milestone">15.4</span> (2022-09-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. @@ -963,7 +1054,7 @@ If you applied customizations to any of the affected analyzers or if you current Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab SAST Security Code Scan analyzer scans .NET code for security vulnerabilities. @@ -1020,7 +1111,7 @@ For further details, see [the deprecation issue for this change](https://gitlab. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to [scan for security vulnerabilities](https://docs.gitlab.com/ee/user/application_security/). @@ -1048,7 +1139,7 @@ See the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564 Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Secure and Protect stages will be bumping the major versions of their analyzers in tandem with the GitLab 15.0 release. This major bump will enable a clear delineation for analyzers, between: @@ -1093,7 +1184,7 @@ Specifically, the following are being deprecated and will no longer be updated a Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Although not recommended or documented, it was possible to deploy a gRPC-aware proxy between Gitaly and @@ -1117,7 +1208,7 @@ the [relevant epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/463). Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To simplify setting a test coverage pattern, in GitLab 15.0 the @@ -1136,7 +1227,7 @@ testing coverage results in merge requests. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The vulnerability check feature is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. @@ -1157,7 +1248,7 @@ The new security approvals feature is similar to vulnerability check. For exampl Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in GitLab 9.0, and will be removed in GitLab 16.0. If you still use these variables, be sure to change to the replacement [predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) which are functionally identical: @@ -1186,7 +1277,7 @@ The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in G Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `projectFingerprint` field in the [PipelineSecurityReportFinding](https://docs.gitlab.com/ee/api/graphql/reference/index.html#pipelinesecurityreportfinding) @@ -1203,7 +1294,7 @@ exposed in the UUID field. Data previously available in the projectFingerprint f Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `started` field in the [iterations API](https://docs.gitlab.com/ee/api/iterations.html#list-project-iterations) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `current` field (already available) which aligns with the naming for other time-based entities, such as milestones. @@ -1342,7 +1433,7 @@ and will remove it in GitLab 15.0 Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users can search for relevant logs in GitLab. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, we don't have a recommended solution for logging within GitLab. For more information, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). @@ -1356,7 +1447,7 @@ The logging features in GitLab allow users to install the ELK stack (Elasticsear Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. By displaying data stored in a Prometheus instance, GitLab allows users to view performance metrics. GitLab also displays visualizations of these metrics in dashboards. The user can connect to a previously-configured external Prometheus instance, or set up Prometheus as a GitLab Managed App. @@ -1428,7 +1519,7 @@ in the Vulnerability Report. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Exporting Sidekiq metrics and health checks using a single process and port is deprecated. @@ -1472,19 +1563,23 @@ Current users of the Static Site Editor can view the [documentation](https://doc Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users can navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab is deprecated in GitLab 14.7, and scheduled for removal in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). </div> -<div class="deprecation removal-150"> +<div class="deprecation removal-150 breaking-change"> ### `artifacts:reports:cobertura` keyword Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + Currently, test coverage visualizations in GitLab only support Cobertura reports. Starting 15.0, the `artifacts:reports:cobertura` keyword will be replaced by [`artifacts:reports:coverage_report`](https://gitlab.com/gitlab-org/gitlab/-/issues/344533). Cobertura will be the @@ -1499,7 +1594,7 @@ only supported report file in 15.0, but this is the first step towards GitLab su Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `merged_by` field in the [merge request API](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `merge_user` field (already present in GraphQL) which more correctly identifies who merged a merge request when performing actions (merge when pipeline succeeds, add to merge train) other than a simple merge. @@ -1518,7 +1613,7 @@ The `merged_by` field in the [merge request API](https://docs.gitlab.com/ee/api/ Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 15.0 we are going to limit the number of characters in CI/CD job names to 255. Any pipeline with job names that exceed the 255 character limit will stop working after the 15.0 release. @@ -1532,7 +1627,7 @@ In GitLab 15.0 we are going to limit the number of characters in CI/CD job names Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We deprecated legacy names for approval status of license policy (blacklisted, approved) in the `managed_licenses` API but they are still used in our API queries and responses. They will be removed in 15.0. @@ -1548,7 +1643,7 @@ If you are using our License Compliance API you should stop using the `approved` Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines that use these keywords will stop working, so you must switch to `stage` and `stages`, which have the same behavior. @@ -1562,7 +1657,7 @@ The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The API Fuzzing configuration snippet is now being generated client-side and does not require an @@ -1578,7 +1673,7 @@ which isn't being used in GitLab anymore. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As of 14.6 bundler-audit is being deprecated from Dependency Scanning. It will continue to be in our CI/CD template while deprecated. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. @@ -1599,7 +1694,7 @@ If you have explicitly excluded bundler-audit using DS_EXCLUDED_ANALYZERS you wi Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 15.0, you can no longer change an instance (shared) runner to a project (specific) runner. @@ -1617,7 +1712,7 @@ Administrators who need to add runners for multiple projects can register a runn Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. @@ -1633,7 +1728,7 @@ In GitLab 15.0 and later, the default value for this configuration option will c Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. A request to the API for `/api/v4/projects/:id/packages` returns a paginated result of packages. Each package lists all of its pipelines in this response. This is a performance concern, as it's possible for a package to have hundreds or thousands of associated pipelines. @@ -1649,7 +1744,7 @@ In milestone 16.0, we will remove the `pipelines` attribute from the API respons Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-04-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Runner REST and GraphQL API endpoints will not return `paused` or `active` as a status in GitLab 16.0. @@ -1669,7 +1764,7 @@ When checking if a runner is `paused`, API users are advised to check the boolea Planned removal: GitLab <span class="removal-milestone">15.6</span> (2022-11-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The certificate-based integration with Kubernetes will be [deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). As a GitLab SaaS customer, on new namespaces, you will no longer be able to integrate GitLab and your cluster using the certificate-based approach as of GitLab 15.0. The integration for current users will be enabled per namespace. The integrations are expected to be switched off completely on GitLab SaaS around 2022 November 22. @@ -1690,7 +1785,7 @@ GitLab self-managed customers can still use the feature [with a feature flag](ht Planned removal: GitLab <span class="removal-milestone">16.0</span> (2023-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The certificate-based integration with Kubernetes [will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). @@ -1713,7 +1808,7 @@ For updates and details about this deprecation, follow [this epic](https://gitla Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. @@ -1727,7 +1822,7 @@ Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). @@ -1743,7 +1838,7 @@ The `GET /groups/:id/registry/repositories` endpoint will remain, but won't retu Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. @@ -1759,7 +1854,7 @@ If you monitor Value Stream Analytics metrics and rely on the date filter, to av Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). @@ -1775,7 +1870,7 @@ In milestone 15.0, we will completely remove `Version` from `PackageType`. Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GraphQL API field `defaultMergeCommitMessageWithDescription` has been deprecated and will be removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template. @@ -1789,7 +1884,7 @@ The GraphQL API field `defaultMergeCommitMessageWithDescription` has been deprec Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We added a feature flag because [GitLab-#11582](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) changed how public groups use the Dependency Proxy. Prior to this change, you could use the Dependency Proxy without authentication. The change requires authentication to use the Dependency Proxy. @@ -1805,7 +1900,7 @@ In milestone 15.0, we will remove the feature flag entirely. Moving forward, you Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: @@ -1824,7 +1919,7 @@ To mitigate possible performance problems, we will remove the `versions` field's Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. `gitlab-ctl promote-db` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. @@ -1838,7 +1933,7 @@ In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Ge Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures. @@ -1869,7 +1964,7 @@ Starting in 14.5 we are providing packages for openSUSE Leap 15.3, and will stop Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are now deprecated and will be removed in GitLab 15.0. @@ -1887,7 +1982,7 @@ dramatically slow down GitLab instances. For this reason, they are being removed Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. [GitLab Serverless](https://docs.gitlab.com/ee/user/project/clusters/serverless/) is a feature set to support Knative-based serverless development with automatic deployments and monitoring. @@ -1903,7 +1998,7 @@ We decided to remove the GitLab Serverless features as they never really resonat Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html) @@ -1921,7 +2016,7 @@ This deprecation mainly impacts users compiling GitLab from source because Omnib Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `omniauth-kerberos` gem will be removed in our next major release, GitLab 15.0. @@ -1970,7 +2065,7 @@ This will result in the rename of the sub-chart: `gitlab/task-runner` to `gitlab Planned removal: GitLab <span class="removal-milestone">15.6</span> (2022-11-22) -With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support.html#gitaly-and-nfs) for further information. +With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) for further information. Gitaly Cluster offers tremendous benefits for our customers such as: @@ -1989,7 +2084,7 @@ We encourage customers currently using NFS for Git repositories to plan their mi Planned removal: GitLab <span class="removal-milestone">15.0</span> (2022-05-22) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The OAuth implicit grant authorization flow will be removed in our next major release, GitLab 15.0. Any applications that use OAuth implicit grant should switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). diff --git a/doc/update/index.md b/doc/update/index.md index e9fa2321450..d4412d85355 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -12,11 +12,6 @@ GitLab version is, if you're upgrading to a major version, and so on. Make sure to read the whole page as it contains information related to every upgrade method. -NOTE: -Upgrade GitLab to the latest available patch release, for example `13.8.8` rather than `13.8.0`. -This includes [versions you must stop at on the upgrade path](#upgrade-paths) as there may -be fixes for issues relating to the upgrade process. - The [maintenance policy documentation](../policy/maintenance.md) has additional information about upgrading, including: @@ -46,9 +41,8 @@ There are also instructions when you want to ### Installation from source -- [Upgrading Community Edition and Enterprise Edition from - source](upgrading_from_source.md) - The guidelines for upgrading Community - Edition and Enterprise Edition from source. +- [Upgrading Community Edition and Enterprise Edition from source](upgrading_from_source.md) - + The guidelines for upgrading Community Edition and Enterprise Edition from source. - [Patch versions](patch_versions.md) guide includes the steps needed for a patch version, such as 13.2.0 to 13.2.1, and apply to both Community and Enterprise Editions. @@ -89,7 +83,7 @@ Background migrations and batched migrations are not the same, so you should che complete before updating. Decrease the time required to complete these migrations by increasing the number of -[Sidekiq workers](../administration/operations/extra_sidekiq_processes.md) +[Sidekiq workers](../administration/sidekiq/extra_sidekiq_processes.md) that can process jobs in the `background_migration` queue. ### Background migrations @@ -118,13 +112,13 @@ sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::Ba For GitLab 14.0-14.9: ```shell -sudo gitlab-rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' ``` For GitLab 14.10 and later: ```shell -sudo gitlab-rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' +sudo gitlab-rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' ``` **For installations from source:** @@ -133,14 +127,14 @@ For GitLab 14.0-14.9: ```shell cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.failed.count' ``` For GitLab 14.10 and later: ```shell cd /home/git/gitlab -sudo -u git -H bundle exec rails runner -e production 'Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' +sudo -u git -H bundle exec rails runner -e production 'puts Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:failed).count' ``` ### Batched background migrations @@ -333,7 +327,14 @@ sudo -u git -H bundle exec rake gitlab:elastic:list_pending_migrations ### What do you do if your Advanced Search migrations are stuck? -See [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). +In GitLab 15.0, an Advanced Search migration named `DeleteOrphanedCommit` can be permanently stuck +in a pending state across upgrades. This issue +[is corrected in GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89539). + +If you are a self-managed customer who uses GitLab 15.0 with Advanced Search, you will experience performance degradation. +To clean up the migration, upgrade to 15.1 or later. + +For other Advanced Search migrations stuck in pending, see [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). ### What do you do for the error `Elasticsearch version not compatible` @@ -381,6 +382,12 @@ accordingly, while also consulting the `8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> [`12.10.14`](#12100) -> `13.0.14` -> [`13.1.11`](#1310) -> [`13.8.8`](#1388) -> [`13.12.15`](#13120) -> [`14.0.12`](#1400) -> [`14.3.6`](#1430) -> [`14.9.5`](#1490) -> [`14.10.Z`](#14100) -> [`15.0.Z`](#1500) -> [latest `15.Y.Z`](https://gitlab.com/gitlab-org/gitlab/-/releases) +NOTE: +When not explicitly specified, upgrade GitLab to the latest available patch +release rather than the first patch release, for example `13.8.8` instead of `13.8.0`. +This includes versions you must stop at on the upgrade path as there may +be fixes for issues relating to the upgrade process. + The following table, while not exhaustive, shows some examples of the supported upgrade paths. Additional steps between the mentioned versions are possible. We list the minimally necessary steps only. @@ -458,13 +465,13 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. -### 15.2.0 (unreleased) +### 15.2.0 - GitLab installations that have multiple web nodes should be [upgraded to 15.1](#1510) before upgrading to 15.2 (and later) due to a configuration change in Rails that can result in inconsistent ETag key generation. -- Some Sidekiq workers were renamed in this release. To avoid any disruption, [run the Rake tasks to migrate any pending jobs](../raketasks/sidekiq_job_migration.md#future-jobs) before starting the upgrade to GitLab 15.2.0. +- Some Sidekiq workers were renamed in this release. To avoid any disruption, [run the Rake tasks to migrate any pending jobs](../administration/sidekiq/sidekiq_job_migration.md#future-jobs) before starting the upgrade to GitLab 15.2.0. ### 15.1.0 @@ -1033,8 +1040,8 @@ In 13.1.0, you must upgrade to either: Failure to do so results in internal errors in the Gitaly service in some RPCs due to the use of the new `--end-of-options` Git flag. -Additionally, in GitLab 13.1.0, the version of [Rails was upgraded from 6.0.3 to -6.0.3.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33454). +Additionally, in GitLab 13.1.0, the version of +[Rails was upgraded from 6.0.3 to 6.0.3.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33454). The Rails upgrade included a change to CSRF token generation which is not backwards-compatible - GitLab servers with the new Rails version generate CSRF tokens that are not recognizable by GitLab servers diff --git a/doc/update/package/convert_to_ee.md b/doc/update/package/convert_to_ee.md index 9f8e56c460c..2bf6d2c580b 100644 --- a/doc/update/package/convert_to_ee.md +++ b/doc/update/package/convert_to_ee.md @@ -22,6 +22,8 @@ that may require Support intervention. The steps can be summed up to: +1. Make a [GitLab backup](../../raketasks/backup_gitlab.md). + 1. Find the currently installed GitLab version: **For Debian/Ubuntu** @@ -67,7 +69,7 @@ The steps can be summed up to: If you want to use `dpkg`/`rpm` instead of `apt-get`/`yum`, go through the first step to find the current GitLab version, then follow [Update using a manually-downloaded package](index.md#upgrade-using-a-manually-downloaded-package), - and then [add your license](../../user/admin_area/license.md). + and then [add your license](../../user/admin_area/license.md). 1. Install the `gitlab-ee` package. The install automatically uninstalls the `gitlab-ce` package on your GitLab server. `reconfigure` @@ -114,5 +116,7 @@ The steps can be summed up to: sudo rm /etc/yum.repos.d/gitlab_gitlab-ce.repo ``` +1. Optional. [Set up the Elasticsearch integration](../../integration/advanced_search/elasticsearch.md) to enable [Advanced Search](../../user/search/advanced_search.md). + That's it! You can now use GitLab Enterprise Edition! To update to a newer version, follow [Update using the official repositories](index.md#upgrade-using-the-official-repositories). diff --git a/doc/update/package/index.md b/doc/update/package/index.md index bf1154d1cf5..12a8b6f3190 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -196,20 +196,6 @@ see how to [upgrade to a later version](../../administration/docs_self_host.md#u ## Troubleshooting -### GitLab 13.7 and later unavailable on Amazon Linux 2 - -Amazon Linux 2 is not an [officially supported operating system](../../administration/package_information/supported_os.md). -However, in past the [official package installation script](https://packages.gitlab.com/gitlab/gitlab-ee/install) -installed the `el/6` package repository if run on Amazon Linux. From GitLab 13.7, we no longer -provide `el/6` packages so administrators must run the [installation script](https://packages.gitlab.com/gitlab/gitlab-ee/install) -again to update the repository to `el/7`: - -```shell -curl --silent "https://packages.gitlab.com/install/repositories/gitlab/gitlab-ee/script.rpm.sh" | sudo bash -``` - -See the [epic on support for GitLab on Amazon Linux 2](https://gitlab.com/groups/gitlab-org/-/epics/2195) for the latest details on official Amazon Linux 2 support. - ### Get the status of a GitLab installation ```shell diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 2374856ff0c..0947bab855f 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -14,7 +14,7 @@ General notes: - If possible, we recommend you test out the upgrade in a test environment before updating your production instance. Ideally, your test environment should mimic your production environment as closely as possible. -- If [working with Support](https://about.gitlab.com/support/scheduling-upgrade-assistance.html) +- If [working with Support](https://about.gitlab.com/support/scheduling-upgrade-assistance/) to create your plan, share details of your architecture, including: - How is GitLab installed? - What is the operating system of the node? @@ -125,12 +125,11 @@ to your instance and then upgrade it for any relevant features you're using. - Account for any [version-specific update instructions](index.md#version-specific-upgrading-instructions). - Account for any [version-specific changes](package/index.md#version-specific-changes). - Check the [OS compatibility with the target GitLab version](../administration/package_information/supported_os.md). -- Due to background migrations, plan to pause any further upgrades after upgrading - to a new major version. +- Due to background migrations, plan to pause before any further upgrades. [All migrations must finish running](index.md#checking-for-background-migrations-before-upgrading) before the next upgrade. - If available in your starting version, consider - [turning on maintenance mode](../administration/maintenance_mode/) during the + [turning on maintenance mode](../administration/maintenance_mode/index.md) during the upgrade. - About PostgreSQL: - On the top bar, select **Menu > Admin**, and look for the version of @@ -167,6 +166,10 @@ If you're using Geo: After updating GitLab, upgrade your runners to match [your new GitLab version](https://docs.gitlab.com/runner/#gitlab-runner-versions). +#### GitLab agent for Kubernetes + +If you have Kubernetes clusters connected with GitLab, [upgrade your GitLab agents for Kubernetes](../user/clusters/agent/install/index.md#update-the-agent-version) to match your new GitLab version. + #### Elasticsearch After updating GitLab, you may have to upgrade @@ -185,7 +188,7 @@ If anything doesn't go as planned: - [`kubesos`](https://gitlab.com/gitlab-com/support/toolbox/kubesos/) if you installed GitLab using the Helm Charts. - For support: - - [Contact GitLab Support](https://support.gitlab.com/hc) and, + - [Contact GitLab Support](https://support.gitlab.com) and, if you have one, your Technical Account Manager. - If [the situation qualifies](https://about.gitlab.com/support/#definitions-of-support-impact) and [your plan includes emergency support](https://about.gitlab.com/support/#priority-support), diff --git a/doc/update/removals.md b/doc/update/removals.md index fa5c016d3ab..cdb35b5faa0 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -55,7 +55,7 @@ The minimum supported browser versions are: ### API: `stale` status returned instead of `offline` or `not_connected` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Runner [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints have changed in 15.0. @@ -68,7 +68,7 @@ The `not_connected` status is no longer valid. It was replaced with `never_conta ### Audit events for repository push events WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are removed as of GitLab 15.0. @@ -81,7 +81,7 @@ Please note that we will add high-volume audit events in the future as part of [ ### Background upload for object storage WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` has been removed in GitLab 15.0. @@ -127,7 +127,7 @@ This workaround will be dropped, so we encourage migrating to consolidated objec ### Container Network and Host Security WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). @@ -144,7 +144,7 @@ For additional context, or to provide feedback regarding this change, please ref ### Container registry authentication with htpasswd WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. @@ -177,7 +177,7 @@ The following `geo:db:*` tasks have been removed from GitLab 15.0 and have been ### DS_DEFAULT_ANALYZERS environment variable WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We are removing the `DS_DEFAULT_ANALYZERS` environment variable from Dependency Scanning on May 22, 2022 in 15.0. After this removal, this variable's value will be ignored. To configure which analyzers to run with the default configuration, you should use the `DS_EXCLUDED_ANALYZERS` variable instead. @@ -185,7 +185,7 @@ We are removing the `DS_DEFAULT_ANALYZERS` environment variable from Dependency ### Dependency Scanning default Java version changed to 17 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. For Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency Scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_JAVA_VERSION` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). Please note that consequently the default version of Gradle is now 7.3.3. @@ -193,7 +193,7 @@ For Dependency Scanning, the default version of Java that the scanner expects wi ### ELK stack logging WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users could search for relevant logs in GitLab directly. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, this feature is no longer available. For more information on the future of logging and observability, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). @@ -201,7 +201,7 @@ The logging features in GitLab allow users to install the ELK stack (Elasticsear ### Elasticsearch 6.8.x in GitLab 15.0 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Elasticsearch 6.8 support has been removed in GitLab 15.0. Elasticsearch 6.8 has reached [end of life](https://www.elastic.co/support/eol). @@ -213,7 +213,7 @@ View the [version requirements](https://docs.gitlab.com/ee/integration/elasticse ### End of support for Python 3.6 in Dependency Scanning WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. For those using Dependency Scanning for Python projects, we are removing support for the default `gemnasium-python:2` image which uses Python 3.6, as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). @@ -221,7 +221,7 @@ For those using Dependency Scanning for Python projects, we are removing support ### External status check API breaking changes WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to @@ -243,7 +243,7 @@ To align with this change, API calls to list external status checks also return ### GitLab Serverless WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. All functionality related to GitLab Serverless was deprecated in GitLab 14.3 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to explore using the following technologies with GitLab CI/CD: @@ -256,7 +256,7 @@ For additional context, or to provide feedback regarding this change, please ref ### Gitaly nodes in virtual storage WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Configuring the Gitaly nodes directly in the virtual storage's root configuration object has been deprecated in GitLab 13.12 and is no longer supported in GitLab 15.0. You must move the Gitaly nodes under the `'nodes'` key as described in [the Praefect configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#praefect). @@ -264,7 +264,7 @@ Configuring the Gitaly nodes directly in the virtual storage's root configuratio ### GraphQL permissions change for Package settings WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. @@ -281,7 +281,7 @@ The issue for this removal is [GitLab-#350682](https://gitlab.com/gitlab-org/git ### Jaeger integration WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users could previously navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab was deprecated in GitLab 14.7, and removed in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). @@ -289,7 +289,7 @@ Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distr ### Known host required for GitLab Runner SSH executor WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml`. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. @@ -307,7 +307,7 @@ We have now removed the deprecated legacy names for approval status of license p ### Move Gitaly Cluster Praefect `database_host_no_proxy` and `database_port_no_proxy configs` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Gitaly Cluster configuration keys for `praefect['database_host_no_proxy']` and `praefect['database_port_no_proxy']` are replaced with `praefect['database_direct_host']` and `praefect['database_direct_port']`. @@ -315,7 +315,7 @@ The Gitaly Cluster configuration keys for `praefect['database_host_no_proxy']` a ### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and is removed from GitLab Shell in GitLab 15.0. @@ -323,7 +323,7 @@ The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks. ### OAuth implicit grant WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The OAuth implicit grant authorization flow is no longer supported. Any applications that use OAuth implicit grant must switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). @@ -331,7 +331,7 @@ The OAuth implicit grant authorization flow is no longer supported. Any applicat ### OAuth tokens without an expiration WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab no longer supports OAuth tokens [without an expiration](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens). @@ -341,7 +341,7 @@ Any existing token without an expiration has one automatically generated and app ### Optional enforcement of SSH expiration WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Disabling SSH expiration enforcement is unusual from a security perspective and could create unusual situations where an expired @@ -351,7 +351,7 @@ expiration on all SSH keys. ### Optional enforcement of personal access token expiration WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Allowing expired personal access tokens to be used is unusual from a security perspective and could create unusual situations where an @@ -376,7 +376,7 @@ If you rely on Java 8 being present in the analyzer environment, you must take a ### Pipelines field from the version field WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: @@ -389,7 +389,7 @@ To mitigate possible performance problems, we will remove the `versions` field's ### Pseudonymizer WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Pseudonymizer feature is generally unused, can cause production issues with large databases, and can interfere with object storage development. @@ -398,7 +398,7 @@ It was removed in GitLab 15.0. ### Request profiling WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. [Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/request_profiling.html) has been removed in GitLab 15.0. @@ -412,7 +412,7 @@ For more information, check the [summary section of the deprecation issue](https ### Required pipeline configurations in Premium tier WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. [Required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) helps to define and mandate organization-wide pipeline configurations and is a requirement at an executive and organizational level. To align better with our [pricing philosophy](https://about.gitlab.com/company/pricing/#three-tiers), this feature is removed from the Premium tier in GitLab 15.0. This feature continues to be available in the GitLab Ultimate tier. @@ -427,7 +427,7 @@ This change also helps GitLab remain consistent in our tiering strategy with the ### Retire-JS Dependency Scanning tool WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We have removed support for retire.js from Dependency Scanning as of May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. @@ -437,7 +437,7 @@ If you have explicitly excluded retire.js using the `DS_EXCLUDED_ANALYZERS` vari ### Runner status `not_connected` API value WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Runner REST and GraphQL [API](https://docs.gitlab.com/ee/api/runners.html#runners-api) endpoints @@ -470,7 +470,7 @@ If you rely on .NET 2.1 support being present in the analyzer image by default, ### SUSE Linux Enterprise Server 12 SP2 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. @@ -496,7 +496,7 @@ For further details, see [the deprecation issue for this change](https://gitlab. ### Self-managed certificate-based integration with Kubernetes feature flagged WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 15.0 the certificate-based integration with Kubernetes will be disabled by default. @@ -512,7 +512,7 @@ For updates and details, follow [this epic](https://gitlab.com/groups/gitlab-org ### Sidekiq configuration for metrics and health checks WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 15.0, you can no longer serve Sidekiq metrics and health checks over a single address and port. @@ -536,7 +536,7 @@ The Static Site Editor was deprecated in GitLab 14.7 and the feature is being re ### Support for `gitaly['internal_socket_dir']` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Gitaly introduced a new directory that holds all runtime data Gitaly requires to operate correctly. This new directory replaces the old internal socket directory, and consequentially the usage of `gitaly['internal_socket_dir']` was deprecated in favor of `gitaly['runtime_dir']`. @@ -544,7 +544,7 @@ Gitaly introduced a new directory that holds all runtime data Gitaly requires to ### Support for legacy format of `config/database.yml` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The syntax of [GitLab's database](https://docs.gitlab.com/omnibus/settings/database.html) @@ -558,7 +558,7 @@ Instructions are available [in the source update documentation](https://docs.git ### Test coverage project CI/CD setting WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To specify a test coverage pattern, in GitLab 15.0 the @@ -571,7 +571,7 @@ To set test coverage parsing, use the project’s `.gitlab-ci.yml` file by provi ### The `promote-db` command is no longer available from `gitlab-ctl` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. The `gitlab-ctl promote-db` command has been removed in GitLab 15.0. @@ -579,7 +579,7 @@ In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Ge ### Update to the Container Registry group-level API WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). @@ -589,7 +589,7 @@ The `GET /groups/:id/registry/repositories` endpoint will remain, but won't retu ### Versions from `PackageType` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). @@ -599,7 +599,7 @@ In GitLab 15.0, we will completely remove `Version` from `PackageType`. ### Vulnerability Check WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The vulnerability check feature was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. @@ -614,13 +614,17 @@ The new security approvals feature is similar to vulnerability check. For exampl ### `Managed-Cluster-Applications.gitlab-ci.yml` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `Managed-Cluster-Applications.gitlab-ci.yml` CI/CD template is being removed. If you need an alternative, try the [Cluster Management project template](https://gitlab.com/gitlab-org/gitlab/-/issues/333610) instead. If your are not ready to move, you can copy the [last released version](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/v14.10.1/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml) of the template into your project. ### `artifacts:reports:cobertura` keyword +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + As of GitLab 15.0, the [`artifacts:reports:cobertura`](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscobertura-removed) keyword has been [replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/344533) by [`artifacts:reports:coverage_report`](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report). @@ -629,7 +633,7 @@ Cobertura is the only supported report file, but this is the first step towards ### `defaultMergeCommitMessageWithDescription` GraphQL API field WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GraphQL API field `defaultMergeCommitMessageWithDescription` has been removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template. @@ -637,7 +641,7 @@ The GraphQL API field `defaultMergeCommitMessageWithDescription` has been remove ### `dependency_proxy_for_private_groups` feature flag WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. A feature flag was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in GitLab 13.7 as part of the change to require authentication to use the Dependency Proxy. Before GitLab 13.7, you could use the Dependency Proxy without authentication. @@ -647,7 +651,7 @@ In GitLab 15.0, we will remove the feature flag, and you must always authenticat ### `omniauth-kerberos` gem WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `omniauth-kerberos` gem is no longer supported. This gem has not been maintained and has very little usage. Therefore, we @@ -660,7 +664,7 @@ We are not removing Kerberos SPNEGO integration. We are removing the old passwor ### `promote-to-primary-node` command from `gitlab-ctl` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` has been removed in GitLab 15.0. @@ -668,7 +672,7 @@ In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Ge ### `push_rules_supersede_code_owners` feature flag WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `push_rules_supersede_code_owners` feature flag has been removed in GitLab 15.0. From now on, push rules will supersede the `CODEOWNERS` file. The code owners feature is no longer available for access control. @@ -676,7 +680,7 @@ The `push_rules_supersede_code_owners` feature flag has been removed in GitLab 1 ### `type` and `types` keyword from CI/CD configuration WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `type` and `types` CI/CD keywords is removed in GitLab 15.0, so pipelines that use these keywords fail with a syntax error. Switch to `stage` and `stages`, which have the same behavior. @@ -684,7 +688,7 @@ The `type` and `types` CI/CD keywords is removed in GitLab 15.0, so pipelines th ### bundler-audit Dependency Scanning tool WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal, Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. @@ -696,7 +700,7 @@ If you have explicitly excluded bundler-audit using the `DS_EXCLUDED_ANALYZERS` ### Permissions change for downloading Composer dependencies WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. @@ -708,7 +712,7 @@ Downloading Composer dependencies without authentication is deprecated in GitLab ### Integrated error tracking disabled by default WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.4, GitLab released an integrated error tracking backend that replaces Sentry. This feature caused database performance issues. In GitLab 14.9, integrated error tracking is removed from GitLab.com, and turned off by default in GitLab self-managed. While we explore the future development of this feature, please consider switching to the Sentry backend by [changing your error tracking to Sentry in your project settings](https://docs.gitlab.com/ee/operations/error_tracking.html#sentry-error-tracking). @@ -779,7 +783,7 @@ The minimum supported browser versions are: ### Auto Deploy CI template v1 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.0, we will update the [Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/stages.html#auto-deploy) CI template to the latest version. This includes new features, bug fixes, and performance improvements with a dependency on the v2 [auto-deploy-image](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image). Auto Deploy CI template v1 is deprecated going forward. @@ -789,7 +793,7 @@ Since the v1 and v2 versions are not backward-compatible, your project might enc ### Breaking changes to Terraform CI template WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab 14.0 renews the Terraform CI template to the latest version. The new template is set up for the GitLab Managed Terraform state, with a dependency on the GitLab `terraform-images` image, to provide a good user experience around GitLab's Infrastructure-as-Code features. @@ -799,7 +803,7 @@ The current stable and latest templates are not compatible, and the current late ### Code Quality RuboCop support changed WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. By default, the Code Quality feature has not provided support for Ruby 2.6+ if you're using the Code Quality template. To better support the latest versions of Ruby, the default RuboCop version is updated to add support for Ruby 2.4 through 3.0. As a result, support for Ruby 2.1, 2.2, and 2.3 is removed. You can re-enable support for older versions by [customizing your configuration](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality.html#rubocop-errors). @@ -809,7 +813,7 @@ Relevant Issue: [Default `codeclimate-rubocop` engine does not support Ruby 2.6+ ### Container Scanning Engine Clair WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Clair, the default container scanning engine, was deprecated in GitLab 13.9 and is removed from GitLab 14.0 and replaced by Trivy. We advise customers who are customizing variables for their container scanning job to [follow these instructions](https://docs.gitlab.com/ee/user/application_security/container_scanning/#change-scanners) to ensure that their container scanning jobs continue to work. @@ -817,7 +821,7 @@ Clair, the default container scanning engine, was deprecated in GitLab 13.9 and ### DAST default template stages WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.0, we've removed the stages defined in the current `DAST.gitlab-ci.yml` template to avoid the situation where the template overrides manual changes made by DAST users. We're making this change in response to customer issues where the stages in the template cause problems when used with customized DAST configurations. Because of this removal, `gitlab-ci.yml` configurations that do not specify a `dast` stage must be updated to include this stage. @@ -825,7 +829,7 @@ In GitLab 14.0, we've removed the stages defined in the current `DAST.gitlab-ci. ### DAST environment variable renaming and removal WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab 13.8 renamed multiple environment variables to support their broader usage in different workflows. In GitLab 14.0, the old variables have been permanently removed and will no longer work. Any configurations using these variables must be updated to the new variable names. Any scans using these variables in GitLab 14.0 and later will fail to be configured correctly. These variables are: @@ -842,7 +846,7 @@ GitLab 13.8 renamed multiple environment variables to support their broader usag ### Default Browser Performance testing job renamed in GitLab 14.0 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Browser Performance Testing has run in a job named `performance` by default. With the introduction of [Load Performance Testing](https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html) in GitLab 13.2, this naming could be confusing. To make it clear which job is running [Browser Performance Testing](https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html), the default job name is changed from `performance` to `browser_performance` in the template in GitLab 14.0. @@ -852,7 +856,7 @@ Relevant Issue: [Rename default Browser Performance Testing job](https://gitlab. ### Default DAST spider begins crawling at target URL WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab 14.0, DAST has removed the current method of resetting the scan to the hostname when starting to spider. Prior to GitLab 14.0, the spider would not begin at the specified target path for the URL but would instead reset the URL to begin crawling at the host root. GitLab 14.0 changes the default for the new variable `DAST_SPIDER_START_AT_HOST` to `false` to better support users' intention of beginning spidering and scanning at the specified target URL, rather than the host root URL. This change has an added benefit: scans can take less time, if the specified path does not contain links to the entire site. This enables easier scanning of smaller sections of an application, rather than crawling the entire app during every scan. @@ -860,7 +864,7 @@ In GitLab 14.0, DAST has removed the current method of resetting the scan to the ### Default branch name for new repositories now `main` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Every Git repository has an initial branch, which is named `master` by default. It's the first branch to be created automatically when you create a new repository. Future [Git versions](https://lore.kernel.org/git/pull.656.v4.git.1593009996.gitgitgadget@gmail.com/) will change the default branch name in Git from `master` to `main`. In coordination with the Git project and the broader community, [GitLab has changed the default branch name](https://gitlab.com/gitlab-org/gitlab/-/issues/223789) for new projects on both our SaaS (GitLab.com) and self-managed offerings starting with GitLab 14.0. This will not affect existing projects. @@ -872,7 +876,7 @@ For more information, check out our [blog post](https://about.gitlab.com/blog/20 ### Dependency Scanning WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As mentioned in [13.9](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#deprecations-for-dependency-scanning) and [this blog post](https://about.gitlab.com/blog/2021/02/08/composition-analysis-14-deprecations-and-removals/) several removals for Dependency Scanning take effect. @@ -884,7 +888,7 @@ Previously, to prevent the Gemnasium analyzers to fetch the advisory database at ### Deprecated GraphQL fields WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In accordance with our [GraphQL deprecation and removal process](https://docs.gitlab.com/ee/api/graphql/#deprecation-process), the following fields that were deprecated prior to 13.7 are [fully removed in 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/267966): @@ -899,7 +903,7 @@ In accordance with our [GraphQL deprecation and removal process](https://docs.gi ### DevOps Adoption API Segments WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The first release of the DevOps Adoption report had a concept of **Segments**. Segments were [quickly removed from the report](https://gitlab.com/groups/gitlab-org/-/epics/5251) because they introduced an additional layer of complexity on top of **Groups** and **Projects**. Subsequent iterations of the DevOps Adoption report focus on comparing adoption across groups rather than segments. GitLab 14.0 removes all references to **Segments** [from the GraphQL API](https://gitlab.com/gitlab-org/gitlab/-/issues/324414) and replaces them with **Enabled groups**. @@ -907,7 +911,7 @@ The first release of the DevOps Adoption report had a concept of **Segments**. S ### Disk source configuration for GitLab Pages WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab Pages [API-based configuration](https://docs.gitlab.com/ee/administration/pages/#gitlab-api-based-configuration) has been available since GitLab 13.0. It replaces the unsupported `disk` source configuration removed in GitLab 14.0, which can no longer be chosen. You should stop using `disk` source configuration, and move to `gitlab` for an API-based configuration. To migrate away from the 'disk' source configuration, set `gitlab_pages['domain_config_source'] = "gitlab"` in your `/etc/gitlab/gitlab.rb` file. We recommend you migrate before updating to GitLab 14.0, to identify and troubleshoot any potential problems before upgrading. @@ -915,7 +919,7 @@ GitLab Pages [API-based configuration](https://docs.gitlab.com/ee/administration ### Experimental prefix in Sidekiq queue selector options WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab supports a [queue selector](https://docs.gitlab.com/ee/administration/operations/extra_sidekiq_processes.html#queue-selector) to run only a subset of background jobs for a given process. When it was introduced, this option had an 'experimental' prefix (`experimental_queue_selector` in Omnibus, `experimentalQueueSelector` in Helm charts). @@ -925,7 +929,7 @@ As announced in the [13.6 release post](https://about.gitlab.com/releases/2020/1 ### External Pipeline Validation Service Code Changes WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. For self-managed instances using the experimental [external pipeline validation service](https://docs.gitlab.com/ee/administration/external_pipeline_validation.html), the range of error codes GitLab accepts will be reduced. Currently, pipelines are invalidated when the validation service returns a response code from `400` to `499`. In GitLab 14.0 and later, pipelines will be invalidated for the `406: Not Accepted` response code only. @@ -933,7 +937,7 @@ For self-managed instances using the experimental [external pipeline validation ### Geo Foreign Data Wrapper settings WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As [announced in GitLab 13.3](https://about.gitlab.com/releases/2020/08/22/gitlab-13-3-released/#geo-foreign-data-wrapper-settings-deprecated), the following configuration settings in `/etc/gitlab/gitlab.rb` have been removed in 14.0: @@ -946,7 +950,7 @@ As [announced in GitLab 13.3](https://about.gitlab.com/releases/2020/08/22/gitla ### GitLab OAuth implicit grant WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab is deprecating the [OAuth 2 implicit grant flow](https://docs.gitlab.com/ee/api/oauth2.html#implicit-grant-flow) as it has been removed for [OAuth 2.1](https://oauth.net/2.1/). @@ -956,7 +960,7 @@ Migrate your existing applications to other supported [OAuth2 flows](https://doc ### GitLab Runner helper image in GitLab.com Container Registry WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 14.0, we are now pulling the GitLab Runner [helper image](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#helper-image) from the GitLab Container Registry instead of Docker Hub. Refer to [issue #27218](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27218) for details. @@ -964,7 +968,7 @@ In 14.0, we are now pulling the GitLab Runner [helper image](https://docs.gitlab ### GitLab Runner installation to ignore the `skel` directory WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab Runner 14.0, the installation process will ignore the `skel` directory by default when creating the user home directory. Refer to [issue #4845](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4845) for details. @@ -972,7 +976,7 @@ In GitLab Runner 14.0, the installation process will ignore the `skel` directory ### Gitaly Cluster SQL primary elector WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Now that Praefect supports a [primary election strategy](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#repository-specific-primary-nodes) for each repository, we have removed the `sql` election strategy. @@ -983,7 +987,7 @@ If you had configured the `sql` election strategy, you must follow the [migratio ### Global `SAST_ANALYZER_IMAGE_TAG` in SAST CI template WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. With the maturity of GitLab Secure scanning tools, we've needed to add more granularity to our release process. Previously, GitLab shared a major version number for [all analyzers and tools](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks). This requires all tools to share a major version, and prevents the use of [semantic version numbering](https://semver.org/). In GitLab 14.0, SAST removes the `SAST_ANALYZER_IMAGE_TAG` global variable in our [managed `SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml) CI template, in favor of the analyzer job variable setting the `major.minor` tag in the SAST vendored template. @@ -995,7 +999,7 @@ This deprecation and removal changes our [previously announced plan](https://abo ### Hardcoded `master` in CI/CD templates WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Our CI/CD templates have been updated to no longer use hard-coded references to a `master` branch. In 14.0, they all use a variable that points to your project's configured default branch instead. If your CI/CD pipeline relies on our built-in templates, verify that this change works with your current configuration. For example, if you have a `master` branch and a different default branch, the updates to the templates may cause changes to your pipeline behavior. For more information, [read the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/324131). @@ -1003,7 +1007,7 @@ Our CI/CD templates have been updated to no longer use hard-coded references to ### Helm v2 support WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Helm v2 was [officially deprecated](https://helm.sh/blog/helm-v2-deprecation-timeline/) in November of 2020, with the `stable` repository being [de-listed from the Helm Hub](https://about.gitlab.com/blog/2020/11/09/ensure-auto-devops-work-after-helm-stable-repo/) shortly thereafter. With the release of GitLab 14.0, which will include the 5.0 release of the [GitLab Helm chart](https://docs.gitlab.com/charts/), Helm v2 will no longer be supported. @@ -1013,7 +1017,7 @@ Users of the chart should [upgrade to Helm v3](https://helm.sh/docs/topics/v2_v3 ### Legacy DAST domain validation WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The legacy method of DAST Domain Validation for CI/CD scans was deprecated in GitLab 13.8, and is removed in GitLab 14.0. This method of domain validation only disallows scans if the `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` environment variable is set to `true` in the `gitlab-ci.yml` file, and a `Gitlab-DAST-Permission` header on the site is not set to `allow`. This two-step method required users to opt in to using the variable before they could opt out from using the header. @@ -1023,7 +1027,7 @@ For more information, see the [removal issue](https://gitlab.com/gitlab-org/gitl ### Legacy feature flags WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Legacy feature flags became read-only in GitLab 13.4. GitLab 14.0 removes support for legacy feature flags, so you must migrate them to the [new version](https://docs.gitlab.com/ee/operations/feature_flags.html). You can do this by first taking a note (screenshot) of the legacy flag, then deleting the flag through the API or UI (you don't need to alter the code), and finally create a new Feature Flag with the same name as the legacy flag you deleted. Also, make sure the strategies and environments match the deleted flag. We created a [video tutorial](https://www.youtube.com/watch?v=CAJY2IGep7Y) to help with this migration. @@ -1031,7 +1035,7 @@ Legacy feature flags became read-only in GitLab 13.4. GitLab 14.0 removes suppor ### Legacy fields from DAST report WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As a part of the migration to a common report format for all of the Secure scanners in GitLab, DAST is making changes to the DAST JSON report. Certain legacy fields were deprecated in 13.8 and have been completely removed in 14.0. These fields are `@generated`, `@version`, `site`, and `spider`. This should not affect any normal DAST operation, but does affect users who consume the JSON report in an automated way and use these fields. Anyone affected by these changes, and needs these fields for business reasons, is encouraged to open a new GitLab issue and explain the need. @@ -1041,7 +1045,7 @@ For more information, see [the removal issue](https://gitlab.com/gitlab-org/gitl ### Legacy storage WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As [announced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#planned-removal-of-legacy-storage-in-14.0), [legacy storage](https://docs.gitlab.com/ee/administration/repository_storage_types.html#legacy-storage) has been removed in GitLab 14.0. @@ -1049,7 +1053,7 @@ As [announced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitla ### License Compliance WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 13.0, we deprecated the License-Management CI template and renamed it License-Scanning. We have been providing backward compatibility by warning users of the old template to switch. Now in 14.0, we are completely removing the License-Management CI template. Read about it in [issue #216261](https://gitlab.com/gitlab-org/gitlab/-/issues/216261) or [this blog post](https://about.gitlab.com/blog/2021/02/08/composition-analysis-14-deprecations-and-removals/). @@ -1057,7 +1061,7 @@ In 13.0, we deprecated the License-Management CI template and renamed it License ### Limit projects returned in `GET /groups/:id/` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To improve performance, we are limiting the number of projects returned from the `GET /groups/:id/` API call to 100. A complete list of projects can still be retrieved with the `GET /groups/:id/projects` API call. @@ -1065,7 +1069,7 @@ To improve performance, we are limiting the number of projects returned from the ### Make `pwsh` the default shell for newly-registered Windows Runners WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab Runner 13.2, PowerShell Core support was added to the Shell executor. In 14.0, PowerShell Core, `pwsh` is now the default shell for newly-registered Windows runners. Windows `CMD` will still be available as a shell option for Windows runners. Refer to [issue #26419](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26419) for details. @@ -1073,7 +1077,7 @@ In GitLab Runner 13.2, PowerShell Core support was added to the Shell executor. ### Migrate from `SAST_DEFAULT_ANALYZERS` to `SAST_EXCLUDED_ANALYZERS` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Until GitLab 13.9, if you wanted to avoid running one particular GitLab SAST analyzer, you needed to remove it from the [long string of analyzers in the `SAST.gitlab-ci.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/390afc431e7ce1ac253b35beb39f19e49c746bff/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml#L12) and use that to set the [`SAST_DEFAULT_ANALYZERS`](https://docs.gitlab.com/ee/user/application_security/sast/#docker-images) variable in your project's CI file. If you did this, it would exclude you from future new analyzers because this string hard codes the list of analyzers to execute. We avoid this problem by inverting this variable's logic to exclude, rather than choose default analyzers. @@ -1082,7 +1086,7 @@ Beginning with 13.9, [we migrated](https://gitlab.com/gitlab-org/gitlab/-/blob/1 ### Off peak time mode configuration for Docker Machine autoscaling WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab Runner 13.0, [issue #5069](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/5069), we introduced new timing options for the GitLab Docker Machine executor. In GitLab Runner 14.0, we have removed the old configuration option, [off peak time mode](https://docs.gitlab.com/runner/configuration/autoscale.html#off-peak-time-mode-configuration-deprecated). @@ -1090,7 +1094,7 @@ In GitLab Runner 13.0, [issue #5069](https://gitlab.com/gitlab-org/gitlab-runner ### OpenSUSE Leap 15.1 WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Support for [OpenSUSE Leap 15.1 is being deprecated](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5135). Support for 15.1 will be dropped in 14.0. We are now providing support for openSUSE Leap 15.2 packages. @@ -1098,7 +1102,7 @@ Support for [OpenSUSE Leap 15.1 is being deprecated](https://gitlab.com/gitlab-o ### PostgreSQL 11 support WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab 14.0 requires PostgreSQL 12 or later. It offers [significant improvements](https://www.postgresql.org/about/news/postgresql-12-released-1976/) to indexing, partitioning, and general performance benefits. @@ -1108,7 +1112,7 @@ Starting in GitLab 13.7, all new installations default to PostgreSQL version 12. ### Redundant timestamp field from DORA metrics API payload WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The [deployment frequency project-level API](https://docs.gitlab.com/ee/api/dora4_project_analytics.html#list-project-deployment-frequencies) endpoint has been deprecated in favor of the [DORA 4 API](https://docs.gitlab.com/ee/api/dora/metrics.html), which consolidates all the metrics under one API with the specific metric as a required field. As a result, the timestamp field, which doesn't allow adding future extensions and causes performance issues, will be removed. With the old API, an example response would be `{ "2021-03-01": 3, "date": "2021-03-01", "value": 3 }`. The first key/value (`"2021-03-01": 3`) will be removed and replaced by the last two (`"date": "2021-03-01", "value": 3`). @@ -1116,7 +1120,7 @@ The [deployment frequency project-level API](https://docs.gitlab.com/ee/api/dora ### Release description in the Tags API WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab 14.0 removes support for the release description in the Tags API. You can no longer add a release description when [creating a new tag](https://docs.gitlab.com/ee/api/tags.html#create-a-new-tag). You also can no longer [create](https://docs.gitlab.com/ee/api/tags.html#create-a-new-release) or [update](https://docs.gitlab.com/ee/api/tags.html#update-a-release) a release through the Tags API. Please migrate to use the [Releases API](https://docs.gitlab.com/ee/api/releases/#create-a-release) instead. @@ -1124,7 +1128,7 @@ GitLab 14.0 removes support for the release description in the Tags API. You can ### Ruby version changed in `Ruby.gitlab-ci.yml` WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. By default, the `Ruby.gitlab-ci.yml` file has included Ruby 2.5. @@ -1136,7 +1140,7 @@ Relevant Issue: [Updates Ruby version 2.5 to 3.0](https://gitlab.com/gitlab-org/ ### SAST analyzer `SAST_GOSEC_CONFIG` variable WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. With the release of [SAST Custom Rulesets](https://docs.gitlab.com/ee/user/application_security/sast/#customize-rulesets) in GitLab 13.5 we allow greater flexibility in configuration options for our Go analyzer (GoSec). As a result we no longer plan to support our less flexible [`SAST_GOSEC_CONFIG`](https://docs.gitlab.com/ee/user/application_security/sast/#analyzer-settings) analyzer setting. This variable was deprecated in GitLab 13.10. @@ -1145,7 +1149,7 @@ GitLab 14.0 removes the old `SAST_GOSEC_CONFIG variable`. If you use or override ### Service Templates WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Service Templates are [removed in GitLab 14.0](https://gitlab.com/groups/gitlab-org/-/epics/5672). They were used to apply identical settings to a large number of projects, but they only did so at the time of project creation. @@ -1155,7 +1159,7 @@ While they solved part of the problem, _updating_ those values later proved to b ### Success and failure for finished build metric conversion WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab Runner 13.5, we introduced `failed` and `success` states for a job. To support Prometheus rules, we chose to convert `success/failure` to `finished` for the metric. In 14.0, the conversion has now been removed. Refer to [issue #26900](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26900) for details. @@ -1163,7 +1167,7 @@ In GitLab Runner 13.5, we introduced `failed` and `success` states for a job. To ### Terraform template version WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. As we continuously [develop GitLab's Terraform integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/325312), to minimize customer disruption, we maintain two GitLab CI/CD templates for Terraform: @@ -1182,7 +1186,7 @@ To check the new changes, see the [new "major version" template](https://gitlab. ### Ubuntu 16.04 support WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Ubuntu 16.04 [reached end-of-life in April 2021](https://ubuntu.com/about/release-cycle), and no longer receives maintenance updates. We strongly recommend users to upgrade to a newer release, such as 20.04. @@ -1192,7 +1196,7 @@ GitLab 13.12 will be the last release with Ubuntu 16.04 support. ### Ubuntu 19.10 (Eoan Ermine) package WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. Ubuntu 19.10 (Eoan Ermine) reached end of life on Friday, July 17, 2020. In GitLab Runner 14.0, Ubuntu 19.10 (Eoan Ermine) is no longer available from our package distribution. Refer to [issue #26036](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26036) for details. @@ -1200,7 +1204,7 @@ Ubuntu 19.10 (Eoan Ermine) reached end of life on Friday, July 17, 2020. In GitL ### Unicorn in GitLab self-managed WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. [Support for Unicorn](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6078) has been removed in GitLab 14.0 in favor of Puma. [Puma has a multi-threaded architecture](https://docs.gitlab.com/ee/administration/operations/puma.html) which uses less memory than a multi-process application server like Unicorn. On GitLab.com, we saw a 40% reduction in memory consumption by using Puma. @@ -1208,7 +1212,7 @@ Review the details carefully before upgrading. ### WIP merge requests renamed 'draft merge requests' WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The WIP (work in progress) status for merge requests signaled to reviewers that the merge request in question wasn't ready to merge. We've renamed the WIP feature to **Draft**, a more inclusive and self-explanatory term. **Draft** clearly communicates the merge request in question isn't ready for review, and makes no assumptions about the progress being made toward it. **Draft** also reduces the cognitive load for new users, non-English speakers, and anyone unfamiliar with the WIP acronym. @@ -1216,7 +1220,7 @@ The WIP (work in progress) status for merge requests signaled to reviewers that ### Web Application Firewall (WAF) WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The Web Application Firewall (WAF) was deprecated in GitLab 13.6 and is removed from GitLab 14.0. The WAF had limitations inherent in the architectural design that made it difficult to meet the requirements traditionally expected of a WAF. By removing the WAF, GitLab is able to focus on improving other areas in the product where more value can be provided to users. Users who currently rely on the WAF can continue to use the free and open source [ModSecurity](https://github.com/SpiderLabs/ModSecurity) project, which is independent from GitLab. Additional details are available in the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/271276). @@ -1224,7 +1228,7 @@ The Web Application Firewall (WAF) was deprecated in GitLab 13.6 and is removed ### Windows Server 1903 image support WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 14.0, we have removed Windows Server 1903. Microsoft ended support for this version on 2020-08-12. Refer to [issue #27551](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27551) for details. @@ -1232,7 +1236,7 @@ In 14.0, we have removed Windows Server 1903. Microsoft ended support for this v ### Windows Server 1909 image support WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 14.0, we have removed Windows Server 1909. Microsoft ended support for this version on 2021-05-11. Refer to [issue #27899](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27899) for details. @@ -1240,7 +1244,7 @@ In 14.0, we have removed Windows Server 1909. Microsoft ended support for this v ### `/usr/lib/gitlab-runner` symlink from package WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In GitLab Runner 13.3, a symlink was added from `/user/lib/gitlab-runner/gitlab-runner` to `/usr/bin/gitlab-runner`. In 14.0, the symlink has been removed and the runner is now installed in `/usr/bin/gitlab-runner`. Refer to [issue #26651](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26651) for details. @@ -1248,7 +1252,7 @@ In GitLab Runner 13.3, a symlink was added from `/user/lib/gitlab-runner/gitlab- ### `?w=1` URL parameter to ignore whitespace changes WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To create a consistent experience for users based on their preferences, support for toggling whitespace changes via URL parameter has been removed in GitLab 14.0. @@ -1256,7 +1260,7 @@ To create a consistent experience for users based on their preferences, support ### `CI_PROJECT_CONFIG_PATH` variable WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. The `CI_PROJECT_CONFIG_PATH` [predefined project variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) @@ -1268,7 +1272,7 @@ please update them to use `CI_CONFIG_PATH` instead. ### `FF_RESET_HELPER_IMAGE_ENTRYPOINT` feature flag WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In 14.0, we have deactivated the `FF_RESET_HELPER_IMAGE_ENTRYPOINT` feature flag. Refer to issue [#26679](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26679) for details. @@ -1276,7 +1280,7 @@ In 14.0, we have deactivated the `FF_RESET_HELPER_IMAGE_ENTRYPOINT` feature flag ### `FF_SHELL_EXECUTOR_USE_LEGACY_PROCESS_KILL` feature flag WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. In [GitLab Runner 13.1](https://docs.gitlab.com/runner/executors/shell.html#gitlab-131-and-later), [issue #3376](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3376), we introduced `sigterm` and then `sigkill` to a process in the Shell executor. We also introduced a new feature flag, `FF_SHELL_EXECUTOR_USE_LEGACY_PROCESS_KILL`, so you can use the previous process termination sequence. In GitLab Runner 14.0, [issue #6413](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6413), the feature flag has been removed. @@ -1284,7 +1288,7 @@ In [GitLab Runner 13.1](https://docs.gitlab.com/runner/executors/shell.html#gitl ### `FF_USE_GO_CLOUD_WITH_CACHE_ARCHIVER` feature flag WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab Runner 14.0 removes the `FF_USE_GO_CLOUD_WITH_CACHE_ARCHIVER` feature flag. Refer to [issue #27175](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27175) for details. @@ -1292,7 +1296,7 @@ GitLab Runner 14.0 removes the `FF_USE_GO_CLOUD_WITH_CACHE_ARCHIVER` feature fla ### `secret_detection_default_branch` job WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. To ensure Secret Detection was scanning both default branches and feature branches, we introduced two separate secret detection CI jobs (`secret_detection_default_branch` and `secret_detection`) in our managed [`Secret-Detection.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) template. These two CI jobs created confusion and complexity in the CI rules logic. This deprecation moves the `rule` logic into the `script` section, which then determines how the `secret_detection` job is run (historic, on a branch, commits, etc). @@ -1301,7 +1305,7 @@ If you override or maintain custom versions of `SAST.gitlab-ci.yml` or `Secret-D ### `trace` parameter in `jobs` API WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/contributing/#breaking-changes). +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). Review the details carefully before upgrading. GitLab Runner was updated in GitLab 13.4 to internally stop passing the `trace` parameter to the `/api/jobs/:id` endpoint. GitLab 14.0 deprecates the `trace` parameter entirely for all other requests of this endpoint. Make sure your [GitLab Runner version matches your GitLab version](https://docs.gitlab.com/runner/#gitlab-runner-versions) to ensure consistent behavior. diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index b953194b4cf..55561abc009 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -10,8 +10,7 @@ comments: false NOTE: In the past we used separate documents for upgrading from Community Edition to Enterprise Edition. These documents can be found in the -[`doc/update` directory of Enterprise Edition's source -code](https://gitlab.com/gitlab-org/gitlab/-/tree/11-8-stable-ee/doc/update). +[`doc/update` directory of Enterprise Edition's source code](https://gitlab.com/gitlab-org/gitlab/-/tree/11-8-stable-ee/doc/update). If you want to upgrade the version only, for example 11.8 to 11.9, *without* changing the GitLab edition you are using (Community or Enterprise), see the diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index 8b921f6d0ce..2df11e8f741 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -119,8 +119,8 @@ rm go1.17.10.linux-amd64.tar.gz To check you are running the minimum required Git version, see [Git versions](../install/installation.md#software-requirements). -From GitLab 13.6, we recommend you use the [Git version provided by -Gitaly](https://gitlab.com/gitlab-org/gitaly/-/issues/2729) +From GitLab 13.6, we recommend you use the +[Git version provided by Gitaly](https://gitlab.com/gitlab-org/gitaly/-/issues/2729) that: - Is always at the version required by GitLab. diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index 3cdc6177a4d..0abdd769a6b 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -53,8 +53,8 @@ migrating data. Background migrations are only added in the monthly releases. Certain major/minor releases may require a set of background migrations to be finished. To guarantee this, such a release processes any remaining jobs before continuing the upgrading procedure. While this doesn't require downtime -(if the above conditions are met) we require that you [wait for background -migrations to complete](index.md#checking-for-background-migrations-before-upgrading) +(if the above conditions are met) we require that you +[wait for background migrations to complete](index.md#checking-for-background-migrations-before-upgrading) between each major/minor release upgrade. The time necessary to complete these migrations can be reduced by increasing the number of Sidekiq workers that can process jobs in the @@ -395,7 +395,7 @@ HA. #### In the application node -According to [official Redis documentation](https://redis.io/topics/admin#upgrading-or-restarting-a-redis-instance-without-downtime), +According to [official Redis documentation](https://redis.io/docs/manual/admin/#upgrading-or-restarting-a-redis-instance-without-downtime), the easiest way to update an HA instance using Sentinel is to upgrade the secondaries one after the other, perform a manual failover from current primary (running old version) to a recently upgraded secondary (running a new diff --git a/doc/user/admin_area/broadcast_messages.md b/doc/user/admin_area/broadcast_messages.md index 959331c16de..a6e6a839912 100644 --- a/doc/user/admin_area/broadcast_messages.md +++ b/doc/user/admin_area/broadcast_messages.md @@ -7,7 +7,8 @@ type: reference, howto # Broadcast messages **(FREE SELF)** -> Target roles [introduced](https://gitlab.com/gitlab-org/growth/team-tasks/-/issues/461) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `role_targeted_broadcast_messages`. Disabled by default. +> - Target roles [introduced](https://gitlab.com/gitlab-org/growth/team-tasks/-/issues/461) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `role_targeted_broadcast_messages`. Disabled by default. +> - Theme [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83251) and background color removed in GitLab 14.10. GitLab can display broadcast messages to users of a GitLab instance. There are two types of broadcast messages: diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 5fd44cf8697..c689d61ad68 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -169,6 +169,20 @@ By default, impersonation is enabled. GitLab can be configured to [disable imper  +#### User identities + +> The ability to see a user's SCIM identity was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294608) in GitLab 15.3. + +When using authentication providers, administrators can see the identities for a user: + +1. On the top bar, select **Menu > Admin**. +1. On the left sidebar, select **Overview > Users**. +1. From the list of users, select a user. +1. Select **Identities**. + +This list shows the user's identities, including SCIM identities. Administrators can use this information to troubleshoot SCIM-related issues and confirm +the identities being used for an account. + #### User Permission Export **(PREMIUM SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1772) in GitLab 13.8. @@ -221,7 +235,7 @@ The [Cohorts](user_cohorts.md) tab displays the monthly cohorts of new users and ### Prevent a user from creating groups -By default, users can create groups. To prevent a user from creating groups: +By default, users can create groups. To prevent a user from creating a top level group: 1. On the top bar, select **Menu > Admin**. 1. On the left sidebar, select **Overview > Users** (`/admin/users`). @@ -230,6 +244,8 @@ By default, users can create groups. To prevent a user from creating groups: 1. Clear the **Can create group** checkbox. 1. Select **Save changes**. +It is also possible to [limit which roles can create a subgroup within a group](../group/subgroups/index.md#change-who-can-create-subgroups). + ### Administering Groups You can administer all groups in the GitLab instance from the Admin Area's Groups page. @@ -250,7 +266,7 @@ sort order is by **Last created**. To search for groups by name, enter your criteria in the search field. The group search is case insensitive, and applies partial matching. -To [Create a new group](../group/index.md#create-a-group) select **New group**. +To [Create a new group](../group/manage.md#create-a-group) select **New group**. ### Administering Topics @@ -421,7 +437,7 @@ For multi-node systems we recommend ingesting the logs into services like Elasti The contents of these log files can be useful when troubleshooting a problem. -For details of these log files and their contents, see [Log system](../../administration/logs.md). +For details of these log files and their contents, see [Log system](../../administration/logs/index.md). The content of each log file is listed in chronological order. To minimize performance issues, a maximum 2000 lines of each log file are shown. diff --git a/doc/user/admin_area/merge_requests_approvals.md b/doc/user/admin_area/merge_requests_approvals.md index 526e8cd17da..e090d4e7f88 100644 --- a/doc/user/admin_area/merge_requests_approvals.md +++ b/doc/user/admin_area/merge_requests_approvals.md @@ -38,4 +38,4 @@ Merge request approval settings that can be set at an instance level are: See also the following, which are affected by instance-level rules: - [Project merge request approval rules](../project/merge_requests/approvals/index.md). -- [Group merge request approval settings](../group/index.md#group-merge-request-approval-settings) available in GitLab 13.9 and later. +- [Group merge request approval settings](../group/manage.md#group-merge-request-approval-settings) available in GitLab 13.9 and later. diff --git a/doc/user/admin_area/monitoring/background_migrations.md b/doc/user/admin_area/monitoring/background_migrations.md index 02d32099c63..87374849674 100644 --- a/doc/user/admin_area/monitoring/background_migrations.md +++ b/doc/user/admin_area/monitoring/background_migrations.md @@ -48,6 +48,50 @@ To disable it: Feature.disable(:execute_batched_migrations_on_schedule) ``` +### Pause batched background migrations in GitLab 14.x + +To pause an ongoing batched background migration, use the `disable` command above. +This command causes the migration to complete the current batch, and then wait to start the next batch. + +Use the following database queries to see the state of the current batched background migration: + +1. Obtain the ID of the running migration: + + ```sql + SELECT + id job_class_name, + table_name, + column_name, + job_arguments + FROM batched_background_migrations + WHERE status <> 3; + ``` + +1. Run this query, replacing `XX` with the ID you obtained in the previous step, + to see the status of the migration: + + ```sql + SELECT + started_at, + finished_at, + finished_at - started_at AS duration, + min_value, + max_value, + batch_size, + sub_batch_size + FROM batched_background_migration_jobs + WHERE batched_background_migration_id = XX + ORDER BY id DESC + limit 10; + ``` + +1. Run the query multiple times within a few minutes to ensure no new row has been added. + If no new row has been added, the migration has been paused. + +1. After confirming the migration has paused, restart the migration (using the `enable` + command above) to proceed with the batch when ready. On larger instances, + background migrations can take as long as 48 hours to complete each batch. + ## Automatic batch size optimization > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60133) in GitLab 13.12. diff --git a/doc/user/admin_area/review_abuse_reports.md b/doc/user/admin_area/review_abuse_reports.md index ec8e6f2dda4..a5e7fcb1b8e 100644 --- a/doc/user/admin_area/review_abuse_reports.md +++ b/doc/user/admin_area/review_abuse_reports.md @@ -26,8 +26,8 @@ The notification email address can also be set and retrieved ## Reporting abuse -To find out more about reporting abuse, see [abuse reports user -documentation](../report_abuse.md). +To find out more about reporting abuse, see +[abuse reports user documentation](../report_abuse.md). ## Resolving abuse reports diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index bedd648b3e7..e33cf4a9082 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -318,7 +318,7 @@ nginx['client_max_body_size'] = "200m" ### This repository has exceeded its size limit -If you receive intermittent push errors in your [Rails exceptions log](../../../administration/logs.md#exceptions_jsonlog), like this: +If you receive intermittent push errors in your [Rails exceptions log](../../../administration/logs/index.md#exceptions_jsonlog), like this: ```plaintext Your push has been rejected, because this repository has exceeded its size limit. diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 65712a9a85c..afb937494e0 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -48,7 +48,7 @@ tier. Users can continue to access the features in a paid tier without sharing u ### Features available in 14.4 and later - [Repository size limit](../settings/account_and_limit_settings.md#repository-size-limit). -- [Group access restriction by IP address](../../group/index.md#group-access-restriction-by-ip-address). +- [Group access restriction by IP address](../../group/access_and_permissions.md#restrict-group-access-by-ip-address). NOTE: Registration is not yet required for participation, but may be added in a future milestone. diff --git a/doc/user/admin_area/settings/user_and_ip_rate_limits.md b/doc/user/admin_area/settings/user_and_ip_rate_limits.md index bb3ee64abac..a35cbe5381a 100644 --- a/doc/user/admin_area/settings/user_and_ip_rate_limits.md +++ b/doc/user/admin_area/settings/user_and_ip_rate_limits.md @@ -144,7 +144,7 @@ Note that the bypass only works if the header is set to `1`. Requests that bypassed the rate limiter because of the bypass header are marked with `"throttle_safelist":"throttle_bypass_header"` in -[`production_json.log`](../../../administration/logs.md#production_jsonlog). +[`production_json.log`](../../../administration/logs/index.md#production_jsonlog). To disable the bypass mechanism, make sure the environment variable `GITLAB_THROTTLE_BYPASS_HEADER` is unset or empty. @@ -170,9 +170,9 @@ the allowlist configuration would be `1,53,217`. Requests that bypassed the rate limiter because of the user allowlist are marked with `"throttle_safelist":"throttle_user_allowlist"` in -[`production_json.log`](../../../administration/logs.md#production_jsonlog). +[`production_json.log`](../../../administration/logs/index.md#production_jsonlog). -At application startup, the allowlist is logged in [`auth.log`](../../../administration/logs.md#authlog). +At application startup, the allowlist is logged in [`auth.log`](../../../administration/logs/index.md#authlog). ## Try out throttling settings before enforcing them @@ -208,7 +208,7 @@ non-protected paths can be done by setting To enable dry run mode for all throttles, the variable can be set to `*`. Setting a throttle to dry run mode logs a message to the -[`auth.log`](../../../administration/logs.md#authlog) when it would hit the limit, while letting the +[`auth.log`](../../../administration/logs/index.md#authlog) when it would hit the limit, while letting the request continue as normal. The log message contains an `env` field set to `track`. The `matched` field contains the name of throttle that was hit. diff --git a/doc/user/admin_area/settings/visibility_and_access_controls.md b/doc/user/admin_area/settings/visibility_and_access_controls.md index 8a9db68b34f..118d375da01 100644 --- a/doc/user/admin_area/settings/visibility_and_access_controls.md +++ b/doc/user/admin_area/settings/visibility_and_access_controls.md @@ -20,7 +20,7 @@ To access the visibility and access control options: ## Define which roles can create projects Instance-level protections for project creation define which roles can -[add projects to a group](../../group/index.md#specify-who-can-add-projects-to-a-group) +[add projects to a group](../../group/manage.md#specify-who-can-add-projects-to-a-group) on the instance. To alter which roles have permission to create projects: 1. Sign in to GitLab as a user with Administrator access level. @@ -53,6 +53,7 @@ By default both administrators and anyone with the **Owner** role can delete a p > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2 for groups created after August 12, 2021. > - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) from default delayed project deletion in GitLab 15.1. > - [Enabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89466) in GitLab 15.1. +> - [Disabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95495) in GitLab 15.3. Instance-level protection against accidental deletion of groups and projects. @@ -102,9 +103,9 @@ In GitLab 15.1 and later, delayed group deletion can be enabled by setting **Del Alternatively, projects that are marked for removal can be deleted immediately. To do so: -1. [Restore the project](../../project/settings/#restore-a-project). +1. [Restore the project](../../project/settings/index.md#restore-a-project). 1. Delete the project as described in the - [Administering Projects page](../../admin_area/#administering-projects). + [Administering Projects page](../../admin_area/index.md#administering-projects). ## Configure project visibility defaults diff --git a/doc/user/analytics/img/product_analytics_commits_per_mr_v14_4.png b/doc/user/analytics/img/product_analytics_commits_per_mr_v14_4.png Binary files differdeleted file mode 100644 index 2bfde7beead..00000000000 --- a/doc/user/analytics/img/product_analytics_commits_per_mr_v14_4.png +++ /dev/null diff --git a/doc/user/analytics/img/productivity_analytics_time_to_merge_v14_4.png b/doc/user/analytics/img/productivity_analytics_time_to_merge_v14_4.png Binary files differdeleted file mode 100644 index 0b30aff2c7a..00000000000 --- a/doc/user/analytics/img/productivity_analytics_time_to_merge_v14_4.png +++ /dev/null diff --git a/doc/user/analytics/img/productivity_analytics_trendline_v14_4.png b/doc/user/analytics/img/productivity_analytics_trendline_v14_4.png Binary files differdeleted file mode 100644 index e0b3c54dee2..00000000000 --- a/doc/user/analytics/img/productivity_analytics_trendline_v14_4.png +++ /dev/null diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index f699fa6d0fb..41547430e88 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -8,8 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Instance-level analytics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12077) in GitLab 12.2. - Instance-level analytics make it possible to aggregate analytics across GitLab, so that users can view information across multiple projects and groups in one place. @@ -18,8 +16,7 @@ in one place. ## Group-level analytics -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195979) in GitLab 12.8. -> - Moved to GitLab Premium in 13.9. +> Moved to GitLab Premium in 13.9. GitLab provides several analytics features at the group level. Some of these features require you to use a higher tier than GitLab Free. @@ -50,7 +47,7 @@ You can use GitLab to review analytics at the project level. Some of these featu The following analytics features are available for users to create personalized views: -- [Application Security](../application_security/security_dashboard/#security-center) +- [Application Security](../application_security/security_dashboard/index.md#security-center) Be sure to review the documentation page for this feature for GitLab tier requirements. @@ -124,14 +121,14 @@ To retrieve metrics for change failure rate, use the [GraphQL](../../api/graphql ### Supported DORA metrics in GitLab -| Metric | Level | API | UI chart | Comments | -|---------------------------|-------------------------|-------------------------------------|---------------------------------------|-------------------------------| -| `deployment_frequency` | Project | [GitLab 13.7 and later](../../api/dora/metrics.md) | GitLab 14.8 and later | The previous API endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/323713) in 13.10. | -| `deployment_frequency` | Group | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 13.12 and later | | -| `lead_time_for_changes` | Project | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 13.11 and later | Unit in seconds. Aggregation method is median. | -| `lead_time_for_changes` | Group | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 14.0 and later | Unit in seconds. Aggregation method is median. | -| `time_to_restore_service` | Project and group | [GitLab 14.9 and later](../../api/dora/metrics.md) | GitLab 15.1 and later | Unit in days. Aggregation method is median. | -| `change_failure_rate` | Project and group | [GitLab 14.10 and later](../../api/dora/metrics.md) | GitLab 15.2 and later | Percentage of deployments. | | +| Metric | Level | API | UI chart | Comments | +|---------------------------|-------------------|-----------------------------------------------------|------------------------|----------| +| `deployment_frequency` | Project | [GitLab 13.7 and later](../../api/dora/metrics.md) | GitLab 14.8 and later | The previous API endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/323713) in 13.10. | +| `deployment_frequency` | Group | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 13.12 and later | | +| `lead_time_for_changes` | Project | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 13.11 and later | Unit in seconds. Aggregation method is median. | +| `lead_time_for_changes` | Group | [GitLab 13.10 and later](../../api/dora/metrics.md) | GitLab 14.0 and later | Unit in seconds. Aggregation method is median. | +| `time_to_restore_service` | Project and group | [GitLab 14.9 and later](../../api/dora/metrics.md) | GitLab 15.1 and later | Unit in days. Aggregation method is median. | +| `change_failure_rate` | Project and group | [GitLab 14.10 and later](../../api/dora/metrics.md) | GitLab 15.2 and later | Percentage of deployments. | ## Definitions diff --git a/doc/user/analytics/merge_request_analytics.md b/doc/user/analytics/merge_request_analytics.md index 29f2ec79800..038b2f0c97e 100644 --- a/doc/user/analytics/merge_request_analytics.md +++ b/doc/user/analytics/merge_request_analytics.md @@ -31,12 +31,12 @@ To view merge request analytics: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Analytics > Merge request**. -## View merge requests merged per month +## View the number of merge requests in a date range > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232651) in GitLab 13.3. > - Filtering [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229266) in GitLab 13.4 -To view the number of merge requests merged per month: +To view the number of merge requests merged during a specific date range: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Analytics > Merge request**. diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index be710f8cbd7..f8b28ead155 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -4,103 +4,65 @@ group: Optimize info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Productivity Analytics **(PREMIUM)** +# Productivity analytics **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12079) in GitLab 12.3. +You can use productivity analytics to identify: -Track development velocity with Productivity Analytics. +- Your development velocity based on how long it takes for a merge request to merge. +- The most time consuming merge requests and potential causes. +- Authors, labels, or milestones with the longest time to merge, or most changes. -For many companies, the development cycle is a black box and getting an estimate of how -long, on average, it takes to deliver features is an enormous endeavor. +Use productivity analytics to view the following merge request statistics for your groups: -While [Value Stream Analytics](../analytics/value_stream_analytics.md) focuses on the entire -Software Development Life Cycle (SDLC) process, Productivity Analytics provides a way for Engineering Management to drill down in a systematic way to uncover patterns and causes for success or failure at an individual, project, or group level. +- Amount of time between merge request creation and merge. +- Amount of time between commits, comments, and merge. +- Complexity of changes, like number of lines of code per commit and number of files. -Productivity can slow down for many reasons ranging from degrading codebase to quickly growing teams. To investigate, department or team leaders can start by visualizing the time it takes for merge requests to be merged. +To view merge request data for projects, use [Merge request analytics](../analytics/merge_request_analytics.md). -## Visualizations and metrics +## View productivity analytics -With Productivity Analytics, GitLab users can: +Prerequisite: -- Visualize typical merge request (MR) lifetime and statistics. A histogram shows the distribution of the time elapsed between creating and merging merge requests. -- Drill down into the most time consuming merge requests, select outliers, and filter subsequent charts to investigate potential causes. -- Filter by group, project, author, label, milestone, or a specific date range. For example, filter down to the merge requests of a specific author in a group or project during a milestone or specific date range. -- Measure velocity over time. To observe progress, visualize the trends of each metric from the charts over time. Zoom in on a particular date range if you notice outliers. +- You must have at least the Reporter role for the group. -## Metrics charts +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Analytics > Productivity**. +1. Optional. Filter results: + 1. Select a project from the dropdown list. + 1. To filter results by author, milestone, or label, + select **Filter results...** and enter a value. + 1. To adjust the date range: + - In the **From** field, select a start date. + - In the **To** field, select an end date. -To access the charts, navigate to a group's sidebar and select **Analytics > Productivity Analytics**. -Metrics and visualizations of **merged** merge requests are available on a project or group level. +## View time metrics for merge requests -### Time to merge +Use the following charts in productivity analytics to view the velocity of your merge requests: -The **Time to merge** histogram shows the number of merge requests and the number -of days it took to merge after creation. Select a column to filter subsequent charts. +- **Time to merge**: number of days it took for a +merge requests to merge after they were created. +- **Trendline**: number of merge requests that were merged in a specific time period. - +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Analytics > Productivity**. -### Trendline +To filter time metrics: -The **Trendline** scatterplot shows all merge requests on a certain date, -and the days it took to complete the action and a 30 day rolling median. Select the dropdown to view: +1. To filter the **Trendline** chart, in the **Time to merge** chart, select a column. +1. To view a specific merge request, below the charts, select a merge request from the **List**. -- Time from first commit to first comment. -- Time from first comment until last commit. -- Time from last commit to merge. -- Number of commits per merge request. -- Number of lines of code (LOC) per commit. -- Number of files touched. +## View commit statistics - +To view commit statistics for your group: -### Commits and merge request size +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Analytics > Productivity**. +1. Under the **Trendline** scatterplot, view the commit statistics: + - The left histogram shows the number of hours between commits, comments, and merges. + - The right histogram shows the number of commits and changes per merge request. -Under the **Trendline** scatterplot, the left-side histogram shows -the time taken (in hours) between commits and comments until the merge -request is merged. Select the dropdown to view: +To filter commit statistics: -- Time from first commit to first comment. -- Time from first comment until last commit. -- Time from last commit to merge. - -The right-side histogram shows the size or complexity of a merge request. -Select the dropdown to view: - -- Number of commits per merge request. -- Number of lines of code (LOC) per commit. -- Number of files touched. - - - -### Merge request list - -The **List** table shows a list of merge requests with their respective time duration metrics. - -Sort metrics by: - -- Time from first commit to first comment. -- Time from first comment until last commit. -- Time from last commit to merge. - -Filter metrics by: - -- Number of commits per merge request. -- Number of lines of code per commit. -- Number of files touched. - -## Filter by date range - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13188) in GitLab 12.4. - -You can filter analytics based on a date range. To filter results: - -1. Select a group. -1. Optional. Select a project. -1. Select a date range by using the available date pickers. - -## Permissions - -The **Productivity Analytics** dashboard can be accessed only: - -- On [GitLab Premium](https://about.gitlab.com/pricing/) and above. -- By users with at least the Reporter role. +1. To view different types of commit data, select the dropdown list next to each histogram. +1. To view a specific merge request, below the charts, select a merge request from the **List**. diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index 96236f60417..80e4700c34c 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -1677,7 +1677,7 @@ For more information, see [Offline environments](../offline_deployments/index.md ## Troubleshooting -### Error waiting for API Security 'http://127.0.0.1:5000' to become available +### `Error waiting for API Security 'http://127.0.0.1:5000' to become available` A bug exists in versions of the API Fuzzing analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the DAST API analyzer. @@ -1690,7 +1690,7 @@ If the issue is occurring with versions v1.6.196 or greater, please contact Supp 1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. 1. The `apifuzzer_fuzz` job definition from your `.gitlab-ci.yml` file. -### Error, the OpenAPI document is not valid. Errors were found during validation of the document using the published OpenAPI schema +### `Error, the OpenAPI document is not valid. Errors were found during validation of the document using the published OpenAPI schema` At the start of an API Fuzzing job the OpenAPI Specification is validated against the [published schema](https://github.com/OAI/OpenAPI-Specification/tree/master/schemas). This error is shown when the provided OpenAPI Specification has validation errors. Errors can be introduced when creating an OpenAPI Specification manually, and also when the schema is generated. @@ -1719,7 +1719,7 @@ For OpenAPI Specifications that are generated automatically validation errors ar 1. Alternatively, you can check the log output and look for schema validation warnings. They are prefixed with messages such as `OpenAPI 2.0 schema validation error` or `OpenAPI 3.0.x schema validation error`. Each failed validation provides extra information about `location` and `description`. Correct each of the validation failures and then resubmit the OpenAPI doc. Note that JSON Schema validation message might not be easy to understand. This is why we recommend the use of editors to validate document. 1. Once the validation issues are resolved, re-run your pipeline. -### Failed to start scanner session (version header not found) +### `Failed to start scanner session (version header not found)` The API Fuzzing engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `apifuzzer_fuzz` job. A common cause of this issue is changing the `FUZZAPI_API` variable from its default. @@ -1733,7 +1733,7 @@ The API Fuzzing engine outputs an error message when it cannot establish a conne - Remove the `FUZZAPI_API` variable from the `.gitlab-ci.yml` file. The value will be inherited from the API Fuzzing CI/CD template. We recommend this method instead of manually setting a value. - If removing the variable is not possible, check to see if this value has changed in the latest version of the [API Fuzzing CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml). If so, update the value in the `.gitlab-ci.yml` file. -### Application cannot determine the base URL for the target API +### `Application cannot determine the base URL for the target API` The API Fuzzing analyzer outputs an error message when it cannot determine the target API after inspecting the OpenAPI document. This error message is shown when the target API has not been set in the `.gitlab-ci.yml`file, it is not available in the `environment_url.txt` file, and it could not be computed using the OpenAPI document. @@ -1802,7 +1802,7 @@ To detect and correct elements that don't comply with the OpenAPI specifications | Editor | OpenAPI 2.0 | OpenAPI 3.0.x | OpenAPI 3.1.x | | -- | -- | -- | -- | | [Swagger Editor](https://editor.swagger.io/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{dotted-circle}** YAML, JSON | -| [Stoplight Studio](https://stoplight.io/studio/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | +| [Stoplight Studio](https://stoplight.io/studio) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | If your OpenAPI document is generated manually, load your document in the editor and fix anything that is non-compliant. If your document is generated automatically, load it in your editor to identify the issues in the schema, then go to the application and perform the corrections based on the framework you are using. @@ -1826,7 +1826,7 @@ API Security can still try to consume an OpenAPI document that does not fully co FUZZAPI_OPENAPI_RELAXED_VALIDATION: On ``` -### No operation in the OpenAPI document is consuming any supported media type +### `No operation in the OpenAPI document is consuming any supported media type` API Security uses the specified media types in the OpenAPI document to generate requests. If no request can be created due to the lack of supported media types, then an error will be thrown. @@ -1844,7 +1844,7 @@ API Security uses the specified media types in the OpenAPI document to generate To get support for your particular problem please use the [getting help channels](https://about.gitlab.com/get-help/). The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and API Fuzzing. -Please use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding API fuzzing to ensure it is quickly reviewed by the right people. Please refer to our [review response SLO](../../../development/code_review.md#review-response-slo) to understand when you should receive a response. +Please use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding API fuzzing to ensure it is quickly reviewed by the right people. Please refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. [Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an award emoji and or join the discussion. diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index cf864068e44..7bb3cb4f64c 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -7,7 +7,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Container Scanning **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3672) in GitLab 10.4. +> - Improved support for FIPS [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263482) in GitLab 13.6 by upgrading `CS_MAJOR_VERSION` from `2` to `3`. +> - Integration with Trivy [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322656) in GitLab 13.9 by upgrading `CS_MAJOR_VERSION` from `3` to `4`. +> - Integration with Clair [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/321451) in GitLab 13.9. +> - Default container scanning with Trivy [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61850) in GitLab 14.0. +> - Integration with Grype as an alternative scanner [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326279) in GitLab 14.0. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86092) the major analyzer version from `4` to `5` in GitLab 15.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86783) from GitLab Ultimate to GitLab Free in 15.0. Your application's Docker image may itself be based on Docker images that contain known vulnerabilities. By including an extra Container Scanning job in your pipeline that scans for those @@ -19,7 +25,7 @@ aspects of inspecting the items your code uses. These items typically include ap dependencies that are almost always imported from external sources, rather than sourced from items you wrote yourself. -GitLab offers both Container Scanning and [Dependency Scanning](../dependency_scanning/) +GitLab offers both Container Scanning and [Dependency Scanning](../dependency_scanning/index.md) to ensure coverage for all of these dependency types. To cover as much of your risk area as possible, we encourage you to use all of our security scanners. @@ -68,7 +74,7 @@ information directly in the merge request. To enable container scanning in your pipeline, you need the following: -- Container Scanning runs in the `test` stage, which is available by default. If you redefine the stages in the `.gitlab-ci.yml` file, the `test` stage is required. +- GitLab CI/CD pipeline must include the `test` stage, which is available unless overridden with the [`stages`](../../../ci/yaml/index.md#stages) keyword. - [GitLab Runner](https://docs.gitlab.com/runner/) with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor on Linux/amd64. - Docker `18.09.03` or higher installed on the same computer as the runner. If you're using the @@ -79,7 +85,6 @@ To enable container scanning in your pipeline, you need the following: - If you're using a third-party container registry, you might need to provide authentication credentials through the `DOCKER_USER` and `DOCKER_PASSWORD` [configuration variables](#available-cicd-variables). For more details on how to use these variables, see [authenticate to a remote registry](#authenticate-to-a-remote-registry). -- GitLab CI/CD pipeline must include the `test` stage, which is available unless overridden with the [`stages`](../../../ci/yaml/index.md#stages) keyword. ## Configuration @@ -224,7 +229,7 @@ container_scanning: CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN: "false" ``` -When you enable this feature, you may see [duplicate findings](../terminology/#duplicate-finding) +When you enable this feature, you may see [duplicate findings](../terminology/index.md#duplicate-finding) in the [Vulnerability Report](../vulnerability_report/) if [Dependency Scanning](../dependency_scanning/) is enabled for your project. This happens because GitLab can't automatically deduplicate findings @@ -680,7 +685,7 @@ It's possible to run the [GitLab container scanning tool](https://gitlab.com/git against a Docker container without needing to run it within the context of a CI job. To scan an image directly, follow these steps: -1. Run [Docker Desktop](https://www.docker.com/products/docker-desktop) +1. Run [Docker Desktop](https://www.docker.com/products/docker-desktop/) or [Docker Machine](https://github.com/docker/machine). 1. Run the analyzer's Docker image, passing the image and tag you want to analyze in the @@ -700,101 +705,21 @@ The results are stored in `gl-container-scanning-report.json`. ## Reports JSON format -The container scanning tool emits a JSON report file. For more information, see the -[schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/container-scanning-report-format.json). - -Here's an example container scanning report: - -```json-doc -{ - "version": "14.0.0", - "vulnerabilities": [ - { - "id": "df52bc8ce9a2ae56bbcb0c4ecda62123fbd6f69b", - "category": "container_scanning", - "message": "CVE-2019-3462 in apt-1.4.8", - "description": "Incorrect sanitation of the 302 redirect field in HTTP transport method of apt versions 1.4.8 and earlier can lead to content injection by a MITM attacker, potentially leading to remote code execution on the target machine.", - "severity": "High", - "confidence": "Unknown", - "solution": "Upgrade apt from 1.4.8 to 1.4.9", - "scanner": { - "id": "trivy", - "name": "trivy" - }, - "location": { - "dependency": { - "package": { - "name": "apt" - }, - "version": "1.4.8" - }, - "operating_system": "debian:9.4", - "image": "registry.gitlab.com/gitlab-org/security-products/dast/webgoat-8.0@sha256:bc09fe2e0721dfaeee79364115aeedf2174cce0947b9ae5fe7c33312ee019a4e", - "default_branch_image": "registry.gitlab.com/gitlab-org/security-products/dast/webgoat-8.0:latest" - }, - "identifiers": [ - { - "type": "cve", - "name": "CVE-2019-3462", - "value": "CVE-2019-3462", - "url": "http://www.securityfocus.com/bid/106690" - } - ], - "links": [ - { - "url": "http://www.securityfocus.com/bid/106690" - }, - { - "url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-3462" - }, - { - "url": "https://lists.apache.org/thread.html/8338a0f605bdbb3a6098bb76f666a95fc2b2f53f37fa1ecc89f1146f@%3Cdevnull.infra.apache.org%3E" - }, - { - "url": "https://lists.debian.org/debian-lts-announce/2019/01/msg00013.html" - }, - { - "url": "https://lists.debian.org/debian-lts-announce/2019/01/msg00014.html" - }, - { - "url": "https://security.netapp.com/advisory/ntap-20190125-0002/" - }, - { - "url": "https://usn.ubuntu.com/3863-1/" - }, - { - "url": "https://usn.ubuntu.com/3863-2/" - }, - { - "url": "https://usn.ubuntu.com/usn/usn-3863-1" - }, - { - "url": "https://usn.ubuntu.com/usn/usn-3863-2" - }, - { - "url": "https://www.debian.org/security/2019/dsa-4371" - } - ] - } - ], - "remediations": [] - "scan": { - "scanner": { - "id": "trivy", - "name": "Trivy", - "url": "https://github.com/aquasecurity/trivy/", - "vendor": { - "name": "GitLab" - }, - "version": "0.16.0" - }, - "type": "container_scanning", - "start_time": "2021-04-14T19:45:58", - "end_time": "2021-04-14T19:46:18", - "status": "success" - } -} -``` +The container scanning tool emits JSON reports which the [GitLab Runner](https://docs.gitlab.com/runner/) +recognizes through the [`artifacts:reports`](../../../ci/yaml/#artifactsreports) +keyword in the CI configuration file. + +Once the CI job finishes, the Runner uploads these reports to GitLab, which are then available in +the CI Job artifacts. In GitLab Ultimate, these reports can be viewed in the corresponding [pipeline](../vulnerability_report/pipeline.md) +and become part of the [Vulnerability Report](../vulnerability_report/). + +These reports must follow a format defined in the +[security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/). See: + +- [Latest schema for the container scanning report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/container-scanning-report-format.json). +- [Example container scanning report](https://gitlab.com/gitlab-examples/security/security-reports/-/blob/master/samples/container-scanning.json) + +For more information, see [Security scanner integration](../../../development/integrations/secure.md). ## Security Dashboard @@ -878,27 +803,5 @@ For information on this, see the [general Application Security troubleshooting s ## Changes -- GitLab 13.6 [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263482) better support for - [FIPS](https://csrc.nist.gov/publications/detail/fips/140/2/final) by upgrading the - `CS_MAJOR_VERSION` from `2` to `3`. Version `3` of the `container_scanning` Docker image uses - [`centos:centos8`](https://hub.docker.com/_/centos) - as the new base. It also removes the use of the [start.sh](https://gitlab.com/gitlab-org/security-products/analyzers/klar/-/merge_requests/77) - script and instead executes the analyzer by default. Any customizations made to the - `container_scanning` job's [`before_script`](../../../ci/yaml/index.md#before_script) - and [`after_script`](../../../ci/yaml/index.md#after_script) - blocks may not work with the new version. To roll back to the previous [`alpine:3.11.3`](https://hub.docker.com/_/alpine)-based - Docker image, you can specify the major version through the [`CS_MAJOR_VERSION`](#available-cicd-variables) - variable. -- GitLab 13.9 [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322656) integration with - [Trivy](https://github.com/aquasecurity/trivy) by upgrading `CS_MAJOR_VERSION` from `3` to `4`. -- GitLab 13.9 [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/321451) the integration with - [Clair](https://github.com/quay/clair/). -- GitLab 14.0 [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61850) - an integration with [Trivy](https://github.com/aquasecurity/trivy) - as the default for container scanning, and also [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326279) - an integration with [Grype](https://github.com/anchore/grype) - as an alternative scanner. -- GitLab 15.0 changed the major analyzer version from `4` to `5`. - -Other changes to the container scanning analyzer can be found in the project's +Changes to the container scanning analyzer can be found in the project's [changelog](https://gitlab.com/gitlab-org/security-products/analyzers/container-scanning/-/blob/master/CHANGELOG.md). diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md index ac3b266ad48..154884c16e7 100644 --- a/doc/user/application_security/coverage_fuzzing/index.md +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -367,12 +367,12 @@ vulnerability: ## Troubleshooting -### Error "Unable to extract corpus folder from artifacts zip file" +### Error `Unable to extract corpus folder from artifacts zip file` If you see this error message, and `COVFUZZ_USE_REGISTRY` is set to `true`, ensure that the uploaded corpus file extracts into a folder named `corpus`. -### Error "400 Bad request - Duplicate package is not allowed" +### Error `400 Bad request - Duplicate package is not allowed` If you see this error message when running the fuzzing job with `COVFUZZ_USE_REGISTRY` set to `true`, ensure that duplicates are allowed. For more details, see diff --git a/doc/user/application_security/dast/browser_based.md b/doc/user/application_security/dast/browser_based.md index ffcd496e2c3..e8373b0c0b7 100644 --- a/doc/user/application_security/dast/browser_based.md +++ b/doc/user/application_security/dast/browser_based.md @@ -5,34 +5,42 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, howto --- -# DAST browser-based crawler **(ULTIMATE)** +# DAST browser-based analyzer **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323423) in GitLab 13.12. WARNING: This product is in an early-access stage and is considered a [beta](../../../policy/alpha-beta-support.md#beta-features) feature. -GitLab DAST's new browser-based crawler is a crawl engine built by GitLab to test Single Page Applications (SPAs) and traditional web applications. -Due to the reliance of modern web applications on JavaScript, handling SPAs or applications that are dependent on JavaScript is paramount to ensuring proper coverage of an application for Dynamic Application Security Testing (DAST). +GitLab DAST's browser-based analyzer was built by GitLab to test Single Page Applications (SPAs) and +traditional web applications. It both crawls the web application and analyzes the resulting output +for vulnerabilities. Analysis of modern applications, heavily reliant on JavaScript, is vital to +ensuring DAST coverage. -The browser-based crawler works by loading the target application into a specially-instrumented Chromium browser. A snapshot of the page is taken before a search to find any actions that a user might perform, -such as clicking on a link or filling in a form. For each action found, the crawler executes it, takes a new snapshot, and determines what in the page changed from the previous snapshot. -Crawling continues by taking more snapshots and finding subsequent actions. +The browser-based scanner works by loading the target application into a specially-instrumented +Chromium browser. A snapshot of the page is taken before a search to find any actions that a user +might perform, such as clicking on a link or filling in a form. For each action found, the +browser-based scanner executes it, takes a new snapshot, and determines what in the page changed +from the previous snapshot. Crawling continues by taking more snapshots and finding subsequent +actions. The benefit of scanning by following user actions in a browser is that the crawler can +interact with the target application much like a real user would, identifying complex flows that +traditional web crawlers don't understand. This results in better coverage of the website. -The benefit of crawling by following user actions in a browser is that the crawler can interact with the target application much like a real user would, identifying complex flows that traditional web crawlers don't understand. This results in better coverage of the website. +The browser-based scanner should provide greater coverage for most web applications, compared +with the current DAST AJAX crawler. While both crawlers are +used together with the current DAST scanner, the combination of the browser-based crawler with the +current DAST scanner is much more effective at finding and testing every page in an application. -Using the browser-based crawler should provide greater coverage for most web applications, compared with the current DAST AJAX crawler. The new crawler replaces the AJAX crawler and is specifically designed to maximize crawl coverage in modern web applications. While both crawlers are currently used in conjunction with the existing DAST scanner, the combination of the browser-based crawler with the current DAST scanner is much more effective at finding and testing every page in an application. +## Enable browser-based analyzer -## Enable browser-based crawler - -The browser-based crawler is an extension to the GitLab DAST product. DAST should be included in the CI/CD configuration and the browser-based crawler enabled using CI/CD variables: +To enable the browser-based analyzer: 1. Ensure the DAST [prerequisites](index.md#prerequisites) are met. -1. Include the [DAST CI template](index.md#include-the-dast-template). -1. Set the target website using the `DAST_WEBSITE` CI/CD variable. +1. Include the [DAST CI/CD template](index.md#include-the-dast-template). +1. Set the target website using the [`DAST_WEBSITE` CI/CD variable](index.md#available-cicd-variables). 1. Set the CI/CD variable `DAST_BROWSER_SCAN` to `true`. -An example configuration might look like the following: +Example extract of `.gitlab-ci.yml` file: ```yaml include: @@ -77,13 +85,20 @@ The [DAST variables](index.md#available-cicd-variables) `SECURE_ANALYZERS_PREFIX ## Vulnerability detection -While the browser-based crawler crawls modern web applications efficiently, vulnerability detection is still managed by the standard DAST/Zed Attack Proxy (ZAP) solution. +Vulnerability detection is gradually being migrated from the default Zed Attack Proxy (ZAP) solution +to the browser-based analyzer. For details of the vulnerability detection already migrated, see +[browser-based vulnerability checks](checks/index.md). -The crawler runs the target website in a browser with DAST/ZAP configured as the proxy server. This ensures that all requests and responses made by the browser are passively scanned by DAST/ZAP. -When running a full scan, active vulnerability checks executed by DAST/ZAP do not use a browser. This difference in how vulnerabilities are checked can cause issues that require certain features of the target website to be disabled to ensure the scan works as intended. +The crawler runs the target website in a browser with DAST/ZAP configured as the proxy server. This +ensures that all requests and responses made by the browser are passively scanned by DAST/ZAP. When +running a full scan, active vulnerability checks executed by DAST/ZAP do not use a browser. This +difference in how vulnerabilities are checked can cause issues that require certain features of the +target website to be disabled to ensure the scan works as intended. -For example, for a target website that contains forms with Anti-CSRF tokens, a passive scan works as intended because the browser displays pages and forms as if a user is viewing the page. -However, active vulnerability checks that run in a full scan cannot submit forms containing Anti-CSRF tokens. In such cases, we recommend you disable Anti-CSRF tokens when running a full scan. +For example, for a target website that contains forms with Anti-CSRF tokens, a passive scan works as +intended because the browser displays pages and forms as if a user is viewing the page. However, +active vulnerability checks that run in a full scan cannot submit forms containing Anti-CSRF tokens. +In such cases, we recommend you disable Anti-CSRF tokens when running a full scan. ## Managing scan time diff --git a/doc/user/application_security/dast/checks/16.7.md b/doc/user/application_security/dast/checks/16.7.md index a02fb3a451f..2e6607575db 100644 --- a/doc/user/application_security/dast/checks/16.7.md +++ b/doc/user/application_security/dast/checks/16.7.md @@ -25,8 +25,8 @@ Only three directives are applicable for the `Strict-Transport-Security` header. Note that invalid directives, or the `Strict-Transport-Security` header appearing more than once (if the values are different) is considered invalid. -Prior to adding to this security configuration to your website, it is recommended you review the hstspreload.org [Deployment -Recommendations](https://hstspreload.org/#deployment-recommendations). +Prior to adding to this security configuration to your website, it is recommended you review the hstspreload.org +[Deployment Recommendations](https://hstspreload.org/#deployment-recommendations). ## Details diff --git a/doc/user/application_security/dast/checks/601.1.md b/doc/user/application_security/dast/checks/601.1.md index 60249c2562d..c51b00cdd36 100644 --- a/doc/user/application_security/dast/checks/601.1.md +++ b/doc/user/application_security/dast/checks/601.1.md @@ -30,5 +30,5 @@ then be used to redirect the user, using the 301 response code and `Location` he ## Links -- [OWASP](https://owasp.org/www-project-cheat-sheets/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) +- [OWASP](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) - [CWE](https://cwe.mitre.org/data/definitions/601.html) diff --git a/doc/user/application_security/dast/checks/798.45.md b/doc/user/application_security/dast/checks/798.45.md deleted file mode 100644 index a800063f15d..00000000000 --- a/doc/user/application_security/dast/checks/798.45.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token Finicity Public Key - -## Description - -The response body contains content that matches the pattern of a Finicity Public Key. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.45 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.51.md b/doc/user/application_security/dast/checks/798.51.md deleted file mode 100644 index f131d31ae65..00000000000 --- a/doc/user/application_security/dast/checks/798.51.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token GCP API key - -## Description - -The response body contains content that matches the pattern of a GCP API key. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.51 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.71.md b/doc/user/application_security/dast/checks/798.71.md deleted file mode 100644 index f0bcc43940d..00000000000 --- a/doc/user/application_security/dast/checks/798.71.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token Lob Publishable API Key - -## Description - -The response body contains content that matches the pattern of a Lob Publishable API Key. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.71 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.73.md b/doc/user/application_security/dast/checks/798.73.md deleted file mode 100644 index eae41a49782..00000000000 --- a/doc/user/application_security/dast/checks/798.73.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token Mailgun public validation key - -## Description - -The response body contains content that matches the pattern of a Mailgun public validation key. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.73 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.76.md b/doc/user/application_security/dast/checks/798.76.md deleted file mode 100644 index 87e6364184f..00000000000 --- a/doc/user/application_security/dast/checks/798.76.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token MapBox API token - -## Description - -The response body contains content that matches the pattern of a MapBox API token. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.76 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.79.md b/doc/user/application_security/dast/checks/798.79.md deleted file mode 100644 index 9a580658a72..00000000000 --- a/doc/user/application_security/dast/checks/798.79.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token MessageBird client ID - -## Description - -The response body contains content that matches the pattern of a MessageBird client ID. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.79 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/798.85.md b/doc/user/application_security/dast/checks/798.85.md deleted file mode 100644 index 0726bdc7fd8..00000000000 --- a/doc/user/application_security/dast/checks/798.85.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -stage: Secure -group: Dynamic Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments ---- - -# Exposure of confidential secret or token Nytimes Access Token - -## Description - -The response body contains content that matches the pattern of a Nytimes Access Token. -Exposing this value could allow attackers to gain access to all resources granted by this token. - -## Remediation - -Review the response body content and remove any exposed values. - -## Details - -| ID | Aggregated | CWE | Type | Risk | -|:---|:--------|:--------|:--------|:--------| -| 798.85 | false | 798 | Passive | High | - -## Links - -- [CWE](https://cwe.mitre.org/data/definitions/798.html) diff --git a/doc/user/application_security/dast/checks/index.md b/doc/user/application_security/dast/checks/index.md index cdfebc07ef2..387682318e6 100644 --- a/doc/user/application_security/dast/checks/index.md +++ b/doc/user/application_security/dast/checks/index.md @@ -81,13 +81,11 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.42](798.42.md) | Exposure of confidential secret or token Finicity API token | High | Passive | | [798.43](798.43.md) | Exposure of confidential secret or token Flickr Access Token | High | Passive | | [798.44](798.44.md) | Exposure of confidential secret or token Finnhub Access Token | High | Passive | -| [798.45](798.45.md) | Exposure of confidential secret or token Finicity Public Key | High | Passive | | [798.46](798.46.md) | Exposure of confidential secret or token Flutterwave Secret Key | High | Passive | | [798.47](798.47.md) | Exposure of confidential secret or token Flutterwave Encryption Key | High | Passive | | [798.48](798.48.md) | Exposure of confidential secret or token Frame.io API token | High | Passive | | [798.49](798.49.md) | Exposure of confidential secret or token Freshbooks Access Token | High | Passive | | [798.50](798.50.md) | Exposure of confidential secret or token GoCardless API token | High | Passive | -| [798.51](798.51.md) | Exposure of confidential secret or token GCP API key | High | Passive | | [798.52](798.52.md) | Exposure of confidential secret or token GitHub Personal Access Token | High | Passive | | [798.53](798.53.md) | Exposure of confidential secret or token GitHub OAuth Access Token | High | Passive | | [798.54](798.54.md) | Exposure of confidential secret or token GitHub App Token | High | Passive | @@ -107,21 +105,16 @@ The [DAST browser-based crawler](../browser_based.md) provides a number of vulne | [798.68](798.68.md) | Exposure of confidential secret or token LinkedIn Client ID | High | Passive | | [798.69](798.69.md) | Exposure of confidential secret or token LinkedIn Client secret | High | Passive | | [798.70](798.70.md) | Exposure of confidential secret or token Lob API Key | High | Passive | -| [798.71](798.71.md) | Exposure of confidential secret or token Lob Publishable API Key | High | Passive | | [798.72](798.72.md) | Exposure of confidential secret or token Mailchimp API key | High | Passive | -| [798.73](798.73.md) | Exposure of confidential secret or token Mailgun public validation key | High | Passive | | [798.74](798.74.md) | Exposure of confidential secret or token Mailgun private API token | High | Passive | | [798.75](798.75.md) | Exposure of confidential secret or token Mailgun webhook signing key | High | Passive | -| [798.76](798.76.md) | Exposure of confidential secret or token MapBox API token | High | Passive | | [798.77](798.77.md) | Exposure of confidential secret or token Mattermost Access Token | High | Passive | | [798.78](798.78.md) | Exposure of confidential secret or token MessageBird API token | High | Passive | -| [798.79](798.79.md) | Exposure of confidential secret or token MessageBird client ID | High | Passive | | [798.80](798.80.md) | Exposure of confidential secret or token Netlify Access Token | High | Passive | | [798.81](798.81.md) | Exposure of confidential secret or token New Relic user API Key | High | Passive | | [798.82](798.82.md) | Exposure of confidential secret or token New Relic user API ID | High | Passive | | [798.83](798.83.md) | Exposure of confidential secret or token New Relic ingest browser API token | High | Passive | | [798.84](798.84.md) | Exposure of confidential secret or token npm access token | High | Passive | -| [798.85](798.85.md) | Exposure of confidential secret or token Nytimes Access Token | High | Passive | | [798.86](798.86.md) | Exposure of confidential secret or token Okta Access Token | High | Passive | | [798.87](798.87.md) | Exposure of confidential secret or token Plaid Client ID | High | Passive | | [798.88](798.88.md) | Exposure of confidential secret or token Plaid Secret key | High | Passive | diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index f8aa2e3d1c6..a49dd8fd646 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -351,7 +351,7 @@ include: - template: DAST-API.gitlab-ci.yml variables: - DAST_API_OPENAPI: http://my.api/api-specification.yml + DAST_API_SPECIFICATION: http://my.api/api-specification.yml ``` #### Import API specification from a file @@ -366,7 +366,7 @@ dast: - cp api-specification.yml /zap/wrk/api-specification.yml variables: GIT_STRATEGY: fetch - DAST_API_OPENAPI: api-specification.yml + DAST_API_SPECIFICATION: api-specification.yml ``` #### Full API scan @@ -402,7 +402,7 @@ include: - template: DAST-API.gitlab-ci.yml variables: - DAST_API_OPENAPI: http://api-test.host.com/api-specification.yml + DAST_API_SPECIFICATION: http://api-test.host.com/api-specification.yml DAST_API_HOST_OVERRIDE: api-test.host.com ``` @@ -417,7 +417,7 @@ include: - template: DAST-API.gitlab-ci.yml variables: - DAST_API_OPENAPI: http://api-test.api.com/api-specification.yml + DAST_API_SPECIFICATION: http://api-test.api.com/api-specification.yml DAST_REQUEST_HEADERS: "Authorization: Bearer my.token" ``` @@ -633,8 +633,7 @@ including a large number of false positives. | `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | | `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | | `DAST_API_HOST_OVERRIDE` <sup>1</sup> | string | Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | -| `DAST_API_OPENAPI` | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | -| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/290241)** in GitLab 15.0. Replaced by `DAST_API_OPENAPI`. The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_API_SPECIFICATION` <sup>1</sup> | URL or string | The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | | `DAST_AUTH_REPORT` <sup>2</sup> | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | | `DAST_AUTH_EXCLUDE_URLS` <sup>2</sup> | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Not supported for API scans. | | `DAST_AUTH_URL` <sup>1,2</sup> | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Not supported for API scans. Example: `https://login.example.com`. | @@ -671,7 +670,7 @@ including a large number of false positives. | `DAST_USERNAME` <sup>1,2</sup> | string | The username to authenticate to in the website. Example: `admin` | | `DAST_USERNAME_FIELD` <sup>1,2</sup> | string | The selector of username field at the sign-in HTML form. Example: `name:username` | | `DAST_XML_REPORT` | string | The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. The variable `DAST_API_OPENAPI` must be specified if this is omitted. | +| `DAST_WEBSITE` <sup>1</sup> | URL | The URL of the website to scan. The variable `DAST_API_SPECIFICATION` must be specified if this is omitted. | | `DAST_ZAP_CLI_OPTIONS` | string | ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | | `DAST_ZAP_LOG_CONFIGURATION` | string | Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `logger.httpsender.name=org.parosproxy.paros.network.HttpSender;logger.httpsender.level=debug;logger.sitemap.name=org.parosproxy.paros.model.SiteMap;logger.sitemap.level=debug;` | | `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | @@ -995,8 +994,7 @@ An on-demand scan can be run in active or passive mode: - _Passive mode_ is the default and runs a ZAP Baseline Scan. - _Active mode_ runs a ZAP Full Scan which is potentially harmful to the site being scanned. To - minimize the risk of accidental damage, running an active scan requires a [validated site - profile](#site-profile-validation). + minimize the risk of accidental damage, running an active scan requires a [validated site profile](#site-profile-validation). ### View on-demand DAST scans diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index fdca02267e4..1f86f2ffa49 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -1524,7 +1524,7 @@ For more information, see [Offline environments](../offline_deployments/index.md ## Troubleshooting -### Error waiting for API Security 'http://127.0.0.1:5000' to become available +### `Error waiting for API Security 'http://127.0.0.1:5000' to become available` A bug exists in versions of the DAST API analyzer prior to v1.6.196 that can cause a background process to fail under certain conditions. The solution is to update to a newer version of the DAST API analyzer. @@ -1537,7 +1537,7 @@ If the issue is occurring with versions v1.6.196 or greater, please contact Supp 1. The `gl-api-security-scanner.log` file available as a job artifact. In the right-hand panel of the job details page, select the **Browse** button. 1. The `dast_api` job definition from your `.gitlab-ci.yml` file. -### Failed to start scanner session (version header not found) +### `Failed to start scanner session (version header not found)` The DAST API engine outputs an error message when it cannot establish a connection with the scanner application component. The error message is shown in the job output window of the `dast_api` job. A common cause of this issue is changing the `DAST_API_API` variable from its default. @@ -1551,7 +1551,7 @@ The DAST API engine outputs an error message when it cannot establish a connecti - Remove the `DAST_API_API` variable from the `.gitlab-ci.yml` file. The value will be inherited from the DAST API CI/CD template. We recommend this method instead of manually setting a value. - If removing the variable is not possible, check to see if this value has changed in the latest version of the [DAST API CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml). If so, update the value in the `.gitlab-ci.yml` file. -### Application cannot determine the base URL for the target API +### `Application cannot determine the base URL for the target API` The DAST API engine outputs an error message when it cannot determine the target API after inspecting the OpenAPI document. This error message is shown when the target API has not been set in the `.gitlab-ci.yml` file, it is not available in the `environment_url.txt` file, and it could not be computed using the OpenAPI document. @@ -1612,7 +1612,7 @@ To detect and correct elements that don't comply with the OpenAPI specifications | Editor | OpenAPI 2.0 | OpenAPI 3.0.x | OpenAPI 3.1.x | | -- | -- | -- | -- | | [Swagger Editor](https://editor.swagger.io/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{dotted-circle}** YAML, JSON | -| [Stoplight Studio](https://stoplight.io/studio/) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | +| [Stoplight Studio](https://stoplight.io/studio) | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | **{check-circle}** YAML, JSON | If your OpenAPI document is generated manually, load your document in the editor and fix anything that is non-compliant. If your document is generated automatically, load it in your editor to identify the issues in the schema, then go to the application and perform the corrections based on the framework you are using. @@ -1636,7 +1636,7 @@ API Security can still try to consume an OpenAPI document that does not fully co DAST_API_OPENAPI_RELAXED_VALIDATION: On ``` -### No operation in the OpenAPI document is consuming any supported media type +### `No operation in the OpenAPI document is consuming any supported media type` API Security uses the specified media types in the OpenAPI document to generate requests. If no request can be created due to the lack of supported media types, then an error will be thrown. @@ -1654,7 +1654,7 @@ API Security uses the specified media types in the OpenAPI document to generate To get support for your particular problem please use the [getting help channels](https://about.gitlab.com/get-help/). The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and DAST API. -Please use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding DAST API to ensure it is quickly reviewed by the right people. Please refer to our [review response SLO](../../../development/code_review.md#review-response-slo) to understand when you should receive a response. +Please use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding DAST API to ensure it is quickly reviewed by the right people. Please refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. [Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an award emoji and or join the discussion. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index d0a91ab664e..521bb6adbf0 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -109,6 +109,8 @@ maximum of two directory levels from the repository's root. For example, the `gemnasium-dependency_scanning` job is enabled if a repository contains either `Gemfile`, `api/Gemfile`, or `api/client/Gemfile`, but not if the only supported dependency file is `api/v1/client/Gemfile`. +For Java and Python, when a supported depedency file is detected, Dependency Scanning attempts to build the project and execute some Java or Python commands to get the list of dependencies. For all other projects, the lock file is parsed to obtain the list of dependencies without needing to build the project first. + When a supported dependency file is detected, all dependencies, including transitive dependencies are analyzed. There is no limit to the depth of nested or transitive dependencies that are analyzed. The following languages and dependency managers are supported: @@ -148,14 +150,13 @@ table.supported-languages ul { <th>Language Versions</th> <th>Package Manager</th> <th>Supported files</th> - <th>Analyzer</th> <th><a href="#how-multiple-files-are-processed">Processes multiple files?</a></th> </tr> </thead> <tbody> <tr> <td>Ruby</td> - <td>Not applicable</td> + <td>All versions</td> <td><a href="https://bundler.io/">Bundler</a></td> <td> <ul> @@ -163,23 +164,20 @@ table.supported-languages ul { <li><code>gems.locked</code></li> </ul> </td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> </tr> <tr> <td>PHP</td> - <td>Not applicable</td> + <td>All versions</td> <td><a href="https://getcomposer.org/">Composer</a></td> <td><code>composer.lock</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> </tr> <tr> <td>C</td> - <td rowspan="2">Not applicable</td> + <td rowspan="2">All versions</td> <td rowspan="2"><a href="https://conan.io/">Conan</a></td> <td rowspan="2"><a href="https://docs.conan.io/en/latest/versioning/lockfiles.html"><code>conan.lock</code></a></td> - <td rowspan="2"><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td rowspan="2">Y</td> </tr> <tr> @@ -187,10 +185,9 @@ table.supported-languages ul { </tr> <tr> <td>Go</td> - <td>Not applicable</td> + <td>All versions</td> <td><a href="https://go.dev/">Go</a></td> <td><code>go.sum</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> </tr> <tr> @@ -211,41 +208,36 @@ table.supported-languages ul { <li><code>build.gradle.kts</code></li> </ul> </td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> <td><a href="https://maven.apache.org/">Maven</a></td> <td><code>pom.xml</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> <td rowspan="2">JavaScript</td> - <td>Not applicable</td> + <td>All versions</td> <td><a href="https://www.npmjs.com/">npm</a></td> <td> <ul> - <li><code>package-lock.json</code></li> + <li><code>package-lock.json</code><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-3">3</a></b></sup></li> <li><code>npm-shrinkwrap.json</code></li> </ul> </td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> </tr> <tr> - <td>Not applicable</td> + <td>All versions</td> <td><a href="https://classic.yarnpkg.com/en/">yarn</a></td> <td><code>yarn.lock</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>Y</td> </tr> <tr> <td>.NET</td> - <td rowspan="2">Not applicable</td> + <td rowspan="2">All versions</td> <td rowspan="2"><a href="https://www.nuget.org/">NuGet</a></td> <td rowspan="2"><a href="https://docs.microsoft.com/en-us/nuget/consume-packages/package-references-in-project-files#enabling-lock-file"><code>packages.lock.json</code></a></td> - <td rowspan="2"><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td rowspan="2">Y</td> </tr> <tr> @@ -256,7 +248,6 @@ table.supported-languages ul { <td rowspan="4">3.9</td> <td><a href="https://setuptools.readthedocs.io/en/latest/">setuptools</a></td> <td><code>setup.py</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> @@ -268,7 +259,6 @@ table.supported-languages ul { <li><code>requires.txt</code></li> </ul> </td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> @@ -276,24 +266,21 @@ table.supported-languages ul { <td> <ul> <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile</code></a></li> - <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile.lock</code></a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-3">3</a></b></sup></li> + <li><a href="https://pipenv.pypa.io/en/latest/basics/#example-pipfile-pipfile-lock"><code>Pipfile.lock</code></a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-4">4</a></b></sup></li> </ul> </td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> <td><a href="https://python-poetry.org/">Poetry</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-5">5</a></b></sup></td> <td><code>poetry.lock</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> <tr> <td>Scala</td> - <td>Not applicable</td> - <td><a href="https://www.scala-sbt.org/">sbt</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-4">4</a></b></sup></td> + <td>All versions</td> + <td><a href="https://www.scala-sbt.org/">sbt</a><sup><b><a href="#notes-regarding-supported-languages-and-package-managers-6">6</a></b></sup></td> <td><code>build.sbt</code></td> - <td><a href="https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium">Gemnasium</a></td> <td>N</td> </tr> </tbody> @@ -311,12 +298,18 @@ table.supported-languages ul { <p> Although Gradle with Java 8 is supported, there are other issues such that Android project builds are not supported at this time. Please see the backlog issue <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/336866">Android support for Dependency - Scanning (gemnasium-maven)</a> for more details. Also, Gradle is not supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is enabled. + Scanning (gemnasium-maven)</a> for more details. Also, Gradle is not supported when <a href="https://docs.gitlab.com/ee/development/fips_compliance.html#enable-fips-mode">FIPS mode</a> is enabled. </p> </li> <li> <a id="notes-regarding-supported-languages-and-package-managers-3"></a> <p> + npm is only supported when `lockfileVersion = 1` or `lockfileVersion = 2`. Work to add support for `lockfileVersion = 3` is being tracked in issue <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/365176">GitLab#365176</a>. + </p> + </li> + <li> + <a id="notes-regarding-supported-languages-and-package-managers-4"></a> + <p> The presence of a <code>Pipfile.lock</code> file alone will <i>not</i> trigger the analyzer; the presence of a <code>Pipfile</code> is still required in order for the analyzer to be executed. However, if a <code>Pipfile.lock</code> file is found, it will be used by <code>Gemnasium</code> to scan the exact package versions listed in this file. @@ -328,12 +321,6 @@ table.supported-languages ul { </p> </li> <li> - <a id="notes-regarding-supported-languages-and-package-managers-4"></a> - <p> - Support for <a href="https://www.scala-sbt.org/">sbt</a> 1.3 and above was added in GitLab 13.9. - </p> - </li> - <li> <a id="notes-regarding-supported-languages-and-package-managers-5"></a> <p> Support for <a href="https://python-poetry.org/">Poetry</a> projects with a <code>poetry.lock</code> file was <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/7006">added in GitLab 15.0</a>. @@ -341,6 +328,12 @@ table.supported-languages ul { <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/32774">Poetry's pyproject.toml support for dependency scanning.</a> </p> </li> + <li> + <a id="notes-regarding-supported-languages-and-package-managers-6"></a> + <p> + Support for <a href="https://www.scala-sbt.org/">sbt</a> 1.3 and above was added in GitLab 13.9. + </p> + </li> </ol> <!-- markdownlint-enable MD044 --> @@ -601,7 +594,7 @@ The following variables allow configuration of global dependency scanning settin | `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs to trust. The bundle of certificates provided here is also used by other tools during the scanning process, such as `git`, `yarn`, or `npm`. See [Using a custom SSL CA certificate authority](#using-a-custom-ssl-ca-certificate-authority) for more details. | | `DS_EXCLUDED_ANALYZERS` | Specify the analyzers (by name) to exclude from Dependency Scanning. For more information, see [Dependency Scanning Analyzers](analyzers.md). | | `DS_DEFAULT_ANALYZERS` | This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/287691) in GitLab 14.0 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/333299) in 15.0. Use `DS_EXCLUDED_ANALYZERS` instead. | -| `DS_EXCLUDED_PATHS` | Exclude files and directories from the scan based on the paths. A comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"`. | +| `DS_EXCLUDED_PATHS` | Exclude files and directories from the scan based on the paths. A comma-separated list of patterns. Patterns can be globs (see [`doublestar.Match`](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4@v4.0.2#Match) for supported patterns), or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"`. | | `DS_IMAGE_SUFFIX` | Suffix added to the image name. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354796) in GitLab 14.10.) Automatically set to `"-fips"` when FIPS mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357922) in GitLab 15.0.) | | `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | | `SECURE_LOG_LEVEL` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. Default: `info`. | @@ -924,8 +917,7 @@ Please check the [Release Process documentation](https://gitlab.com/gitlab-org/s ## Contributing to the vulnerability database -You can search the [`gemnasium-db`](https://gitlab.com/gitlab-org/security-products/gemnasium-db) project -to find a vulnerability in the GitLab Advisory Database. +To find a vulnerability, you can search the [`GitLab Advisory Database`](https://advisories.gitlab.com/). You can also [submit new vulnerabilities](https://gitlab.com/gitlab-org/security-products/gemnasium-db/blob/master/CONTRIBUTING.md). ## Running dependency scanning in an offline environment diff --git a/doc/user/application_security/get-started-security.md b/doc/user/application_security/get-started-security.md new file mode 100644 index 00000000000..4c2b971b5fa --- /dev/null +++ b/doc/user/application_security/get-started-security.md @@ -0,0 +1,34 @@ +--- +stage: DevSecOps +group: Technical writing +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Get started with GitLab application security **(ULTIMATE)** + +Complete the following steps to get the most from GitLab application security tools. + +1. Enable [Secret Detection](secret_detection/index.md) scanning for your default branch. +1. Enable [Dependency Scanning](dependency_scanning/index.md) for your default branch so you can start identifying existing + vulnerable packages in your codebase. +1. Add security scans to feature branch pipelines. The same scans should be enabled as are running + on your default branch. Subsequent scans will show only new vulnerabilities by comparing the feature branch to the default branch results. +1. Let your team get comfortable with [vulnerability reports](vulnerability_report/index.md) and + establish a vulnerability triage workflow. +1. Consider creating [labels](../project/labels.md) and [issue boards](../project/issue_board.md) to + help manage issues created from vulnerabilities. Issue boards allow all stakeholders to have a + common view of all issues. +1. Create a [scan result policy](policies/index.md) to limit new vulnerabilities from being merged + into your default branch. +1. Monitor the [Security Dashboard](security_dashboard/index.md) trends to gauge success in + remediating existing vulnerabilities and preventing the introduction of new ones. +1. Enable other scan types such as [SAST](sast/index.md), [DAST](dast/index.md), + [Fuzz testing](coverage_fuzzing/index.md), or [Container Scanning](container_scanning/index.md). + Be sure to add the same scan types to both feature pipelines and default branch pipelines. +1. Use [Compliance Pipelines](../../user/project/settings/index.md#compliance-pipeline-configuration) + or [Scan Execution Policies](policies/scan-execution-policies.md) to enforce required scan types + and ensure separation of duties between security and engineering. +1. Consider enabling [Review Apps](../../development/testing_guide/review_apps.md) to allow for DAST + and [Web API fuzzing](api_fuzzing/index.md) on ephemeral test environments. +1. Enable [operational container scanning](../../user/clusters/agent/vulnerabilities.md) to scan + container images in your production cluster for security vulnerabilities. diff --git a/doc/user/application_security/iac_scanning/index.md b/doc/user/application_security/iac_scanning/index.md index 35968a6361f..16f08de738b 100644 --- a/doc/user/application_security/iac_scanning/index.md +++ b/doc/user/application_security/iac_scanning/index.md @@ -64,7 +64,7 @@ variables: SAST_IMAGE_SUFFIX: '-fips' include: - - template: Security/SAST-IaC.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml ``` ### Making IaC analyzers available to all GitLab tiers @@ -98,11 +98,11 @@ To configure IaC Scanning for a project you can: ### Configure IaC Scanning manually To enable IaC Scanning you must [include](../../../ci/yaml/index.md#includetemplate) the -[`SAST-IaC.latest.gitlab-ci.yml template`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml) provided as part of your GitLab installation. Here is an example of how to include it: +[`SAST-IaC.gitlab-ci.yml template`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml) provided as part of your GitLab installation. Here is an example of how to include it: ```yaml include: - - template: Security/SAST-IaC.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml ``` The included template creates IaC scanning jobs in your CI/CD pipeline and scans @@ -130,3 +130,24 @@ The IaC tool emits a JSON report file in the existing SAST report format. For mo The JSON report file can be downloaded from the CI pipelines page, or the pipelines tab on merge requests by [setting `artifacts: paths`](../../../ci/yaml/index.md#artifactspaths) to `gl-sast-report.json`. For more information see [Downloading artifacts](../../../ci/pipelines/job_artifacts.md). + +## Troubleshooting + +### IaC debug logging + +To help troubleshoot IaC jobs, you can increase the [Secure scanner log verbosity](../sast/index.md#logging-level) +by using a global CI/CD variable set to `debug`: + +```yaml +variables: + SECURE_LOG_LEVEL: "debug" +``` + +### IaC Scanning findings show as `No longer detected` unexpectedly + +If a previously detected finding unexpectedly shows as `No longer detected`, it might +be due to an update to the scanner. An update can disable rules that are found to +be ineffective or false positives, and the findings are marked as `No longer detected`: + +- In GitLab 15.3, [secret detection in the KICS SAST IaC scanner was disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/346181), + so IaC findings in the "Passwords and Secrets" family show as `No longer detected`. diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index e3a419ea771..7c7d5380a24 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -149,19 +149,8 @@ base address for Docker images. You can override this for most scanners by setti The [Container Scanning](container_scanning/index.md) analyzer is an exception, and it does not use the `SECURE_ANALYZERS_PREFIX` variable. To override its Docker image, see -the instructions for [Running container scanning in an offline -environment](container_scanning/index.md#running-container-scanning-in-an-offline-environment). - -### Use security scanning tools with merge request pipelines - -By default, the application security jobs are configured to run for branch pipelines only. -To use them with [merge request pipelines](../../ci/pipelines/merge_request_pipelines.md), -you may need to override the default `rules:` configuration to add: - -```yaml -rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" -``` +the instructions for +[Running container scanning in an offline environment](container_scanning/index.md#running-container-scanning-in-an-offline-environment). ## Default behavior of GitLab security scanning tools @@ -401,8 +390,10 @@ Validation depends on the schema version declared in the security report artifac - If your security report specifies a supported schema version, GitLab uses this version to validate. - If your security report uses a deprecated version, GitLab attempts validation against that version and adds a deprecation warning to the validation result. -- If your security report uses a version that is not supported, GitLab attempts to validate it against the latest schema version available in GitLab. -- If your security report does not specify a schema version, GitLab attempts to validate it against the lastest schema version available in GitLab. Since the `version` property is required, validation always fails in this case, but other validation errors may also be present. +- If your security report uses a supported MAJOR-MINOR version of the report schema but the PATCH version doesn't match any vendored versions, GitLab attempts to validate it against latest vendored PATCH version of the schema. + - Example: security report uses version 14.1.1 but the latest vendored version is 14.1.0. GitLab would validate against schema version 14.1.0. +- If your security report uses a version that is not supported, GitLab attempts to validate it against the latest schema version available in your installation but doesn't ingest the report. +- If your security report does not specify a schema version, GitLab attempts to validate it against the latest schema version available in GitLab. Because the `version` property is required, validation always fails in this case, but other validation errors may also be present. You can always find supported and deprecated schema versions in the [source code](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/parsers/security/validators/schema_validator.rb). diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index 81a9cef885d..53f9c400259 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -137,3 +137,13 @@ See [Scan result policies](scan-result-policies.md). See the [Category Direction page](https://about.gitlab.com/direction/protect/security_orchestration/) for more information on the product direction of security policies within GitLab. + +## Troubleshooting + +### `Branch name does not follow the pattern 'update-policy-<timestamp>'` + +When you create a new security policy or change an existing policy, a new branch is automatically created with the branch name following the pattern `update-policy-<timestamp>`. For example: `update-policy-1659094451`. + +If you have group or instance push rules that do not allow branch name patterns that contain the text `update-policy-<timestamp>`, you will get an error that states `Branch name does not follow the pattern 'update-policy-<timestamp>'`. + +The workaround is to amend your group or instance push rules to allow branches following the pattern `update-policy-` followed by an integer timestamp. diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index 0f9e3343072..cbd64e278c8 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -47,7 +47,7 @@ For an analyzer to be considered Generally Available, it is expected to minimall support the following features: - [Customizable configuration](index.md#available-cicd-variables) -- [Customizable rulesets](index.md#customize-rulesets) +- [Customizable rulesets](customize_rulesets.md#customize-rulesets) - [Scan projects](index.md#supported-languages-and-frameworks) - [Multi-project support](index.md#multi-project-support) - [Offline support](index.md#running-sast-in-an-offline-environment) diff --git a/doc/user/application_security/sast/customize_rulesets.md b/doc/user/application_security/sast/customize_rulesets.md new file mode 100644 index 00000000000..919a3565d88 --- /dev/null +++ b/doc/user/application_security/sast/customize_rulesets.md @@ -0,0 +1,381 @@ +--- +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Customize rulesets **(ULTIMATE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235382) in GitLab 13.5. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/339614) support for +> passthrough chains. Expanded to include additional passthrough types of `file`, `git`, and `url` in GitLab 14.6. +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/235359) support for overriding rules in GitLab 14.8. + +You can customize the default scanning rules provided by our SAST analyzers. +Ruleset customization supports the following that can be used +simultaneously: + +- [Disabling predefined rules](#disable-predefined-analyzer-rules). Available for all analyzers. +- [Overriding predefined rules](#override-predefined-analyzer-rules). Available for all analyzers. +- Modifying the default behavior of a given analyzer by [synthesizing and passing a custom configuration](#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. + +To customize the default scanning rules, create a file containing custom rules. These rules +are passed through to the analyzer's underlying scanner tools. + +To create a custom ruleset: + +1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. +1. Create a custom ruleset file named `sast-ruleset.toml` in the `.gitlab` directory. + +## Disable predefined analyzer rules + +To disable analyzer rules: + +1. Set the `disabled` flag to `true` in the context of a `ruleset` section + +1. In one or more `ruleset.identifier` sub sections, list the rules that you want disabled. Every `ruleset.identifier` section has: + +- a `type` field, to name the predefined rule identifier that the targeted analyzer uses. +- a `value` field, to name the rule to be disabled. + +### Example: Disable predefined rules of SAST analyzers + +In the following example, the disabled rules are assigned to `eslint` +and `sobelow` by matching the `type` and `value` of identifiers: + +```toml +[eslint] + [[eslint.ruleset]] + disable = true + [eslint.ruleset.identifier] + type = "eslint_rule_id" + value = "security/detect-object-injection" + + [[eslint.ruleset]] + disable = true + [eslint.ruleset.identifier] + type = "cwe" + value = "185" + +[sobelow] + [[sobelow.ruleset]] + disable = true + [sobelow.ruleset.identifier] + type = "sobelow_rule_id" + value = "sql_injection" +``` + +Those vulnerabilities containing the provided type and value are now disabled, meaning +they won't be displayed in Merge Request nor the Vulnerability Report. + +## Override predefined analyzer rules + +To override analyzer rules: + +1. In one or more `ruleset.identifier` subsections, list the rules that you want to override. Every `ruleset.identifier` section has: + + - a `type` field, to name the predefined rule identifier that the targeted analyzer uses. + - a `value` field, to name the rule to be overridden. + +1. In the `ruleset.override` context of a `ruleset` section, + provide the keys to override. Any combination of keys can be + overridden. Valid keys are: + + - description + - message + - name + - severity (valid options are: Critical, High, Medium, Low, Unknown, Info) + +### Example: Override predefined rules of SAST analyzers + +Before adding a ruleset, we verify which vulnerability will be overwritten by viewing the [`gl-sast-report.json`](index.md#reports-json-format): + +```json +"identifiers": [ + { + "type": "gosec_rule_id", + "name": "Gosec Rule ID G307", + "value": "G307" + }, + { + "type": "CWE", + "name": "CWE-703", + "value": "703", + "url": "https://cwe.mitre.org/data/definitions/703.html" + } + ] +``` + +In the following example, rules from `gosec` are matched by the `type` +and `value` of identifiers and then overridden: + +```toml +[gosec] + [[gosec.ruleset]] + [gosec.ruleset.identifier] + type = "CWE" + value = "703" + [gosec.ruleset.override] + severity = "Critical" +``` + +If a vulnerability is found with a type `CWE` with a value of `703` then +the vulnerability severity is overwritten to `Critical`. + +## Synthesize a custom configuration + +To create a custom configuration, you can use passthrough chains. + +A passthrough is a single step in a passthrough chain. The passthrough is evaluated +in a sequence to incrementally build a configuration. The configuration is then +passed to the target analyzer. + +A configuration section for an analyzer has the following +parameters: + +| Parameter | Explanation | +| ------------- | ------ | +| `description` | Description about the analyzer configuration section. | +| `targetdir` | The `targetdir` parameter defines the directory where the final configuration is located. If `targetdir` is empty, the analyzer uses a random directory. The maximum size of `targetdir` is 100MB. | +| `validate` | If set to `true`, the target files for passthroughs (`raw`, `file` and `url`) are validated. The validation works for `yaml`, `xml`, `json` and `toml` files. The proper validator is identified based on the extension of the target file. By default, `validate` is set to `false`. | +| `interpolate` | If set to `true`, environment variable interpolation is enabled so that the configuration uses secrets/tokens. We advise using this feature with caution to not leak any secrets. By default, `interpolate` is set to `false`. | +| `timeout` | The total `timeout` for the evaluation of a passthrough chain is set to 60 seconds. If `timeout` is not set, the default timeout is 60 seconds. The timeout cannot exceed 300 seconds. | + +A configuration section can include one or more passthrough sections. The maximum number of passthrough sections is 20. +There are several types of passthroughs: + +| Type | Description | +| ------ | ------ | +| `file` | Use a file that is already available in the Git repository. | +| `raw` | Provide the configuration inline. | +| `git` | Pull the configuration from a remote Git repository. | +| `url` | Fetch the analyzer configuration through HTTP. | + +If multiple passthrough sections are defined in a passthrough chain, their +position in the chain defines the order in which they are evaluated. + +- Passthroughs listed later in the chain sequence have a higher precedence. +- Passthroughs with a higher precedence overwrite (default) and append data + yielded by previous passthroughs. This is useful for cases where you need to + use or modify an existing configuration. + +Configure a passthrough these parameters: + +| Parameter | Explanation | +| ------------ | ----------- | +| `type` | One of `file`, `raw`, `git` or `url`. | +| `target` | The target file that contains the data written by the passthrough evaluation. If no value is provided, a random target file is generated. | +| `mode` | `overwrite`: if `target` exists, overwrites the file; `append`: append to file instead. The default is `overwrite`. | +| `ref` | This option only applies to the `git` passthrough type and contains the name of the branch or the SHA to be used. | +| `subdir` | This option only applies to the `git` passthrough type and can be used to only consider a certain subdirectory of the source Git repository. | +| `value` | For the `file` `url` and `git` types, `value` defines the source location of the file/Git repository; for the `raw` type, `value` carries the raw content to be passed through. | +| `validator` | Can be used to explicitly invoke validators (`xml`, `yaml`, `json`, `toml`) on the target files after the application of a passthrough. Per default, no validator is set. | + +The amount of data generated by a single passthrough is limited to 1MB. + +## Passthrough configuration examples + +### Raw passthrough for nodejs-scan + +Define a custom analyzer configuration. In this example, customized rules are +defined for the `nodejs-scan` scanner: + +```toml +[nodejs-scan] + description = 'custom ruleset for nodejs-scan' + + [[nodejs-scan.passthrough]] + type = "raw" + value = ''' +- nodejs-extensions: + - .js + + template-extensions: + - .new + - .hbs + - '' + + ignore-filenames: +- skip.js + + ignore-paths: + - __MACOSX + - skip_dir + - node_modules + + ignore-extensions: + - .hbs + + ignore-rules: + - regex_injection_dos + - pug_jade_template + - express_xss + +''' +``` + +### File passthrough for Gosec + +Provide the name of the file containing a custom analyzer configuration. In +this example, customized rules for the `gosec` scanner are contained in the +file `gosec-config.json`: + +```toml +[gosec] + description = 'custom ruleset for gosec' + + [[gosec.passthrough]] + type = "file" + value = "gosec-config.json" +``` + +### Passthrough chain for Semgrep + +In the below example, we generate a custom configuration under the `/sgrules` +target directory with a total `timeout` of 60 seconds. + +Several passthrouh types generate a configuration for the target analyzer: + +- Two `git` passthrough sections pull the head of branch + `refs/remotes/origin/test` from the `myrules` Git repository, and revision + `97f7686` from the `sast-rules` Git repository. From the `sast-rules` Git + repository, only data from the `go` subdirectory is considered. + - The `sast-rules` entry has a higher precedence because it appears later in + the configuration. + - If there is a filename collision between files in both repositories, files + from the `sast` repository overwrite files from the `myrules` repository, + as `sast-rules` has higher precedence. +- The `raw` entry creates a file named `insecure.yml` under `/sgrules`. The + full path is `/sgrules/insecure.yml`. +- The `url` entry fetches a configuration made available through a URL and + stores it in the `/sgrules/gosec.yml` file. + +Afterwards, Semgrep is invoked with the final configuration located under +`/sgrules`. + +```toml +[semgrep] + description = 'semgrep custom rules configuration' + targetdir = "/sgrules" + timeout = 60 + + [[semgrep.passthrough]] + type = "git" + value = "https://gitlab.com/user/myrules.git" + ref = "refs/remotes/origin/test" + + [[semgrep.passthrough]] + type = "git" + value = "https://gitlab.com/gitlab-org/secure/gsoc-sast-vulnerability-rules/playground/sast-rules.git" + ref = "97f7686db058e2141c0806a477c1e04835c4f395" + subdir = "go" + + [[semgrep.passthrough]] + type = "raw" + target = "insecure.yml" + value = """ +rules: +- id: "insecure" + patterns: + - pattern: "func insecure() {...}" + message: | + Insecure function insecure detected + metadata: + cwe: "CWE-200: Exposure of Sensitive Information to an Unauthorized Actor" + severity: "ERROR" + languages: + - "go" + """ + + [[semgrep.passthrough]] + type = "url" + value = "https://semgrep.dev/c/p/gosec" + target = "gosec.yml" +``` + +### Interpolation + +The code snippet below shows an example configuration that uses an environment +variable `$GITURL` to access a private repositories with a Git URL. The variable contains +a username and token in the `value` field (for example `https://user:token@url`). +It does not explicitly store credentials in the configuration file. To reduce the risk of leaking secrets through created paths and files, use this feature with caution. + +```toml +[semgrep] + description = 'semgrep custom rules configuration' + targetdir = "/sgrules" + interpolate = true + + [[semgrep.passthrough]] + type = "git" + value = "$GITURL" + ref = "refs/remotes/origin/main" +``` + +### Configure the append mode for passthroughs + +To append data to previous passthroughs, use the `append` mode for the +passthrough types `file`, `url`, and `raw`. + +Passthroughs in `override` mode overwrite files +created when preceding passthroughs in the chain find a naming +collision. If `mode` is set to `append`, a passthrough appends data to the +files created by its predecessors instead of overwriting. + +In the below Semgrep configuration,`/sgrules/insecure.yml` assembles two passthroughs. The rules are: + +- `insecure` +- `secret` + +These rules add a search pattern to the analyzer and extends Semgrep capabilities. + +For passthrough chains we recommend that you enable validation. To enable validation, +you can either: + +- set `validate` to `true` + +- set a passthrough `validator` to `xml`, `json`, `yaml`, or `toml`. + +```toml +[semgrep] + description = 'semgrep custom rules configuration' + targetdir = "/sgrules" + validate = true + + [[semgrep.passthrough]] + type = "raw" + target = "insecure.yml" + value = """ +rules: +- id: "insecure" + patterns: + - pattern: "func insecure() {...}" + message: | + Insecure function insecure detected + metadata: + cwe: "... + severity: "ERROR" + languages: + - "go" +""" + + [[semgrep.passthrough]] + type = "raw" + mode = "append" + target = "insecure.yml" + value = """ +- id: "secret" + patterns: + - pattern-either: + - pattern: "$MASK = \"...\"" + - metavariable-regex: + metavariable: "$MASK" + regex: "(password|pass|passwd|pwd|secret|token)" + message: | + Use of Hard-coded Password + cwe: "..." + severity: "ERROR" + languages: + - "go" +""" +``` diff --git a/doc/user/application_security/sast/img/sast_vulnerability_page_fp_detection_v15_2.png b/doc/user/application_security/sast/img/sast_vulnerability_page_fp_detection_v15_2.png Binary files differnew file mode 100644 index 00000000000..2a3e6e7e45f --- /dev/null +++ b/doc/user/application_security/sast/img/sast_vulnerability_page_fp_detection_v15_2.png diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 92dc795afe5..d4b0d5b972c 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -176,18 +176,18 @@ All open source (OSS) analyzers have been moved to the GitLab Free tier as of Gi Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), as shown in the following table: -| Capability | In Free & Premium | In Ultimate | -|:----------------------------------------------------------------|:--------------------|:-------------------| -| [Configure SAST scanners](#configuration) | **{check-circle}** | **{check-circle}** | -| [Customize SAST settings](#available-cicd-variables) | **{check-circle}** | **{check-circle}** | -| Download [JSON Report](#reports-json-format) | **{check-circle}** | **{check-circle}** | -| See new findings in merge request widget | **{dotted-circle}** | **{check-circle}** | -| [Manage vulnerabilities](../vulnerabilities/index.md) | **{dotted-circle}** | **{check-circle}** | -| [Access the Security Dashboard](../security_dashboard/index.md) | **{dotted-circle}** | **{check-circle}** | -| [Configure SAST in the UI](#configure-sast-in-the-ui) | **{dotted-circle}** | **{check-circle}** | -| [Customize SAST rulesets](#customize-rulesets) | **{dotted-circle}** | **{check-circle}** | -| [Detect False Positives](#false-positive-detection) | **{dotted-circle}** | **{check-circle}** | -| [Track moved vulnerabilities](#advanced-vulnerability-tracking) | **{dotted-circle}** | **{check-circle}** | +| Capability | In Free & Premium | In Ultimate | +|:---------------------------------------------------------------- -|:--------------------|:-------------------| +| [Configure SAST scanners](#configuration) | **{check-circle}** | **{check-circle}** | +| [Customize SAST settings](#available-cicd-variables) | **{check-circle}** | **{check-circle}** | +| Download [JSON Report](#reports-json-format) | **{check-circle}** | **{check-circle}** | +| See new findings in merge request widget | **{dotted-circle}** | **{check-circle}** | +| [Manage vulnerabilities](../vulnerabilities/index.md) | **{dotted-circle}** | **{check-circle}** | +| [Access the Security Dashboard](../security_dashboard/index.md) | **{dotted-circle}** | **{check-circle}** | +| [Configure SAST in the UI](#configure-sast-in-the-ui) | **{dotted-circle}** | **{check-circle}** | +| [Customize SAST rulesets](customize_rulesets.md) | **{dotted-circle}** | **{check-circle}** | +| [Detect False Positives](#false-positive-detection) | **{dotted-circle}** | **{check-circle}** | +| [Track moved vulnerabilities](#advanced-vulnerability-tracking) | **{dotted-circle}** | **{check-circle}** | ## Contribute your scanner @@ -320,382 +320,6 @@ brakeman-sast: SAST_ANALYZER_IMAGE_TAG: "2.21.1" ``` -### Customize rulesets **(ULTIMATE)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235382) in GitLab 13.5. -> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/339614) support for -> passthrough chains. Expanded to include additional passthrough types of `file`, `git`, and `url` in GitLab 14.6. -> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/235359) support for overriding rules in GitLab 14.8. - -You can customize the default scanning rules provided by our SAST analyzers. -Ruleset customization supports the following that can be used -simultaneously: - -- [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). Available for all analyzers. -- [Overriding predefined rules](index.md#override-predefined-analyzer-rules). Available for all analyzers. -- Modifying the default behavior of a given analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). Available for only `nodejs-scan`, `gosec`, and `semgrep`. - -To customize the default scanning rules, create a file containing custom rules. These rules -are passed through to the analyzer's underlying scanner tools. - -To create a custom ruleset: - -1. Create a `.gitlab` directory at the root of your project, if one doesn't already exist. -1. Create a custom ruleset file named `sast-ruleset.toml` in the `.gitlab` directory. - -#### Disable predefined analyzer rules - -To disable analyzer rules: - -1. Set the `disabled` flag to `true` in the context of a `ruleset` section - -1. In one or more `ruleset.identifier` sub sections, list the rules that you want disabled. Every `ruleset.identifier` section has: - -- a `type` field, to name the predefined rule identifier that the targeted analyzer uses. -- a `value` field, to name the rule to be disabled. - -##### Example: Disable predefined rules of SAST analyzers - -In the following example, the disabled rules are assigned to `eslint` -and `sobelow` by matching the `type` and `value` of identifiers: - -```toml -[eslint] - [[eslint.ruleset]] - disable = true - [eslint.ruleset.identifier] - type = "eslint_rule_id" - value = "security/detect-object-injection" - - [[eslint.ruleset]] - disable = true - [eslint.ruleset.identifier] - type = "cwe" - value = "185" - -[sobelow] - [[sobelow.ruleset]] - disable = true - [sobelow.ruleset.identifier] - type = "sobelow_rule_id" - value = "sql_injection" -``` - -Those vulnerabilities containing the provided type and value are now disabled, meaning -they won't be displayed in Merge Request nor the Vulnerability Report. - -#### Override predefined analyzer rules - -To override analyzer rules: - -1. In one or more `ruleset.identifier` subsections, list the rules that you want to override. Every `ruleset.identifier` section has: - - - a `type` field, to name the predefined rule identifier that the targeted analyzer uses. - - a `value` field, to name the rule to be overridden. - -1. In the `ruleset.override` context of a `ruleset` section, - provide the keys to override. Any combination of keys can be - overridden. Valid keys are: - - - description - - message - - name - - severity (valid options are: Critical, High, Medium, Low, Unknown, Info) - -##### Example: Override predefined rules of SAST analyzers - -Before adding a ruleset, we verify which vulnerability will be overwritten by viewing the [`gl-sast-report.json`](#reports-json-format): - -```json -"identifiers": [ - { - "type": "gosec_rule_id", - "name": "Gosec Rule ID G307", - "value": "G307" - }, - { - "type": "CWE", - "name": "CWE-703", - "value": "703", - "url": "https://cwe.mitre.org/data/definitions/703.html" - } - ] -``` - -In the following example, rules from `gosec` are matched by the `type` -and `value` of identifiers and then overridden: - -```toml -[gosec] - [[gosec.ruleset]] - [gosec.ruleset.identifier] - type = "CWE" - value = "703" - [gosec.ruleset.override] - severity = "Critical" -``` - -If a vulnerability is found with a type `CWE` with a value of `703` then -the vulnerability severity is overwritten to `Critical`. - -#### Synthesize a custom configuration - -To create a custom configuration, you can use passthrough chains. - -A passthrough is a single step in a passthrough chain. The passthrough is evaluated -in a sequence to incrementally build a configuration. The configuration is then -passed to the target analyzer. - -A configuration section for an analyzer has the following -parameters: - -| Parameter | Explanation | -| ------------- | ------ | -| `description` | Description about the analyzer configuration section. | -| `targetdir` | The `targetdir` parameter defines the directory where the final configuration is located. If `targetdir` is empty, the analyzer uses a random directory. The maximum size of `targetdir` is 100MB. | -| `validate` | If set to `true`, the target files for passthroughs (`raw`, `file` and `url`) are validated. The validation works for `yaml`, `xml`, `json` and `toml` files. The proper validator is identified based on the extension of the target file. By default, `validate` is set to `false`. | -| `interpolate` | If set to `true`, environment variable interpolation is enabled so that the configuration uses secrets/tokens. We advise using this feature with caution to not leak any secrets. By default, `interpolate` is set to `false`. | -| `timeout` | The total `timeout` for the evaluation of a passthrough chain is set to 60 seconds. If `timeout` is not set, the default timeout is 60 seconds. The timeout cannot exceed 300 seconds. | - -A configuration section can include one or more passthrough sections. The maximum number of passthrough sections is 20. -There are several types of passthroughs: - -| Type | Description | -| ------ | ------ | -| `file` | Use a file that is already available in the Git repository. | -| `raw` | Provide the configuration inline. | -| `git` | Pull the configuration from a remote Git repository. | -| `url` | Fetch the analyzer configuration through HTTP. | - -If multiple passthrough sections are defined in a passthrough chain, their -position in the chain defines the order in which they are evaluated. - -- Passthroughs listed later in the chain sequence have a higher precedence. -- Passthroughs with a higher precedence overwrite (default) and append data - yielded by previous passthroughs. This is useful for cases where you need to - use or modify an existing configuration. - -Configure a passthrough these parameters: - -| Parameter | Explanation | -| ------------ | ----------- | -| `type` | One of `file`, `raw`, `git` or `url`. | -| `target` | The target file that contains the data written by the passthrough evaluation. If no value is provided, a random target file is generated. | -| `mode` | `overwrite`: if `target` exists, overwrites the file; `append`: append to file instead. The default is `overwrite`. | -| `ref` | This option only applies to the `git` passthrough type and contains the name of the branch or the SHA to be used. | -| `subdir` | This option only applies to the `git` passthrough type and can be used to only consider a certain subdirectory of the source Git repository. | -| `value` | For the `file` `url` and `git` types, `value` defines the source location of the file/Git repository; for the `raw` type, `value` carries the raw content to be passed through. | -| `validator` | Can be used to explicitly invoke validators (`xml`, `yaml`, `json`, `toml`) on the target files after the application of a passthrough. Per default, no validator is set. | - -The amount of data generated by a single passthrough is limited to 1MB. - -#### Passthrough configuration examples - -##### Raw passthrough for nodejs-scan - -Define a custom analyzer configuration. In this example, customized rules are -defined for the `nodejs-scan` scanner: - -```toml -[nodejs-scan] - description = 'custom ruleset for nodejs-scan' - - [[nodejs-scan.passthrough]] - type = "raw" - value = ''' -- nodejs-extensions: - - .js - - template-extensions: - - .new - - .hbs - - '' - - ignore-filenames: -- skip.js - - ignore-paths: - - __MACOSX - - skip_dir - - node_modules - - ignore-extensions: - - .hbs - - ignore-rules: - - regex_injection_dos - - pug_jade_template - - express_xss - -''' -``` - -##### File passthrough for Gosec - -Provide the name of the file containing a custom analyzer configuration. In -this example, customized rules for the `gosec` scanner are contained in the -file `gosec-config.json`: - -```toml -[gosec] - description = 'custom ruleset for gosec' - - [[gosec.passthrough]] - type = "file" - value = "gosec-config.json" -``` - -##### Passthrough chain for Semgrep - -In the below example, we generate a custom configuration under the `/sgrules` -target directory with a total `timeout` of 60 seconds. - -Several passthrouh types generate a configuration for the target analyzer: - -- Two `git` passthrough sections pull the head of branch - `refs/remotes/origin/test` from the `myrules` Git repository, and revision - `97f7686` from the `sast-rules` Git repository. From the `sast-rules` Git - repository, only data from the `go` subdirectory is considered. - - The `sast-rules` entry has a higher precedence because it appears later in - the configuration. - - If there is a filename collision between files in both repositories, files - from the `sast` repository overwrite files from the `myrules` repository, - as `sast-rules` has higher precedence. -- The `raw` entry creates a file named `insecure.yml` under `/sgrules`. The - full path is `/sgrules/insecure.yml`. -- The `url` entry fetches a configuration made available through a URL and - stores it in the `/sgrules/gosec.yml` file. - -Afterwards, Semgrep is invoked with the final configuration located under -`/sgrules`. - -```toml -[semgrep] - description = 'semgrep custom rules configuration' - targetdir = "/sgrules" - timeout = 60 - - [[semgrep.passthrough]] - type = "git" - value = "https://gitlab.com/user/myrules.git" - ref = "refs/remotes/origin/test" - - [[semgrep.passthrough]] - type = "git" - value = "https://gitlab.com/gitlab-org/secure/gsoc-sast-vulnerability-rules/playground/sast-rules.git" - ref = "97f7686db058e2141c0806a477c1e04835c4f395" - subdir = "go" - - [[semgrep.passthrough]] - type = "raw" - target = "insecure.yml" - value = """ -rules: -- id: "insecure" - patterns: - - pattern: "func insecure() {...}" - message: | - Insecure function insecure detected - metadata: - cwe: "CWE-200: Exposure of Sensitive Information to an Unauthorized Actor" - severity: "ERROR" - languages: - - "go" - """ - - [[semgrep.passthrough]] - type = "url" - value = "https://semgrep.dev/c/p/gosec" - target = "gosec.yml" -``` - -##### Interpolation - -The code snippet below shows an example configuration that uses an environment -variable `$GITURL` to access a private repositories with a Git URL. The variable contains -a username and token in the `value` field (for example `https://user:token@url`). -It does not explicitly store credentials in the configuration file. To reduce the risk of leaking secrets through created paths and files, use this feature with caution. - -```toml -[semgrep] - description = 'semgrep custom rules configuration' - targetdir = "/sgrules" - interpolate = true - - [[semgrep.passthrough]] - type = "git" - value = "$GITURL" - ref = "refs/remotes/origin/main" -``` - -##### Configure the append mode for passthroughs - -To append data to previous passthroughs, use the `append` mode for the -passthrough types `file`, `url`, and `raw`. - -Passthroughs in `override` mode overwrite files -created when preceding passthroughs in the chain find a naming -collision. If `mode` is set to `append`, a passthrough appends data to the -files created by its predecessors instead of overwriting. - -In the below Semgrep configuration,`/sgrules/insecure.yml` assembles two passthroughs. The rules are: - -- `insecure` -- `secret` - -These rules add a search pattern to the analyzer and extends Semgrep capabilities. - -For passthrough chains we recommend that you enable validation. To enable validation, -you can either: - -- set `validate` to `true` - -- set a passthrough `validator` to `xml`, `json`, `yaml`, or `toml`. - -```toml -[semgrep] - description = 'semgrep custom rules configuration' - targetdir = "/sgrules" - validate = true - - [[semgrep.passthrough]] - type = "raw" - target = "insecure.yml" - value = """ -rules: -- id: "insecure" - patterns: - - pattern: "func insecure() {...}" - message: | - Insecure function insecure detected - metadata: - cwe: "... - severity: "ERROR" - languages: - - "go" -""" - - [[semgrep.passthrough]] - type = "raw" - mode = "append" - target = "insecure.yml" - value = """ -- id: "secret" - patterns: - - pattern-either: - - pattern: "$MASK = \"...\"" - - metavariable-regex: - metavariable: "$MASK" - regex: "(password|pass|passwd|pwd|secret|token)" - message: | - Use of Hard-coded Password - cwe: "..." - severity: "ERROR" - languages: - - "go" -""" -``` - ### False Positive Detection **(ULTIMATE)** > Introduced in GitLab 14.2. @@ -706,6 +330,8 @@ False positive detection is available in a subset of the [supported languages](# - Ruby, in the Brakeman-based analyzer + + ### Advanced vulnerability tracking **(ULTIMATE)** > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5144) in GitLab 14.2. @@ -829,7 +455,7 @@ spotbugs-sast: dependencies: - build variables: - MAVEN_REPO_PATH: ./.m2/repository + MAVEN_REPO_PATH: $CI_PROJECT_DIR/.m2/repository COMPILE: "false" artifacts: reports: @@ -907,7 +533,7 @@ Some analyzers make it possible to filter out vulnerabilities under a given thre | CI/CD variable | Default value | Description | |------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SAST_EXCLUDED_PATHS` | `spec, test, tests, tmp` | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. You might need to exclude temporary directories used by your build tool as these can generate false positives. To exclude paths, copy and paste the default excluded paths, then **add** your own paths to be excluded. If you don't specify the default excluded paths, you will override the defaults and _only_ paths you specify will be excluded from the SAST scans. | +| `SAST_EXCLUDED_PATHS` | `spec, test, tests, tmp` | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs (see [`doublestar.Match`](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4@v4.0.2#Match) for supported patterns), or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. You might need to exclude temporary directories used by your build tool as these can generate false positives. To exclude paths, copy and paste the default excluded paths, then **add** your own paths to be excluded. If you don't specify the default excluded paths, you will override the defaults and _only_ paths you specify will be excluded from the SAST scans. | | `SEARCH_MAX_DEPTH` | 4 | SAST searches the repository to detect the programming languages used, and selects the matching analyzers. Set the value of `SEARCH_MAX_DEPTH` to specify how many directory levels the search phase should span. After the analyzers have been selected, the _entire_ repository is analyzed. | | `SAST_BANDIT_EXCLUDED_PATHS` | | Comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html); For example: `'*/tests/*, */venv/*'` | | `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | @@ -963,6 +589,8 @@ removed, or promoted to regular features at any time. Experimental features available are: - Enable scanning of iOS and Android apps using the [MobSF analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/). +- Disable the following rules in the [Semgrep analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) that are known to cause a high rate of false positives: + - [`eslint.detect-object-injection`](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/6c4764567d9854f5e4a4a35dacf5a68def7fb4c1/rules/eslint.yml#L751-773) #### Enable experimental features diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 02d50b0a857..93c32f998fa 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -200,7 +200,7 @@ Secret Detection can be customized by defining available CI/CD variables: | CI/CD variable | Default value | Description | |-----------------------------------|---------------|-------------| -| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | +| `SECRET_DETECTION_EXCLUDED_PATHS` | "" | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs (see [`doublestar.Match`](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4@v4.0.2#Match) for supported patterns), or file or folder paths (for example, `doc,spec` ). Parent directories also match patterns. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225273) in GitLab 13.3. | | `SECRET_DETECTION_HISTORIC_SCAN` | false | Flag to enable a historic Gitleaks scan. | | `SECRET_DETECTION_IMAGE_SUFFIX` | "" | Suffix added to the image name. If set to `-fips`, `FIPS-enabled` images are used for scan. See [FIPS-enabled images](#fips-enabled-images) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355519) in GitLab 14.10. | | `SECRET_DETECTION_LOG_OPTIONS` | "" | [`git log`](https://git-scm.com/docs/git-log) options used to define commit ranges. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350660) in GitLab 15.1.| @@ -224,9 +224,9 @@ You can customize the default secret detection rules provided with GitLab. Ruleset customization supports the following capabilities that can be used simultaneously: -- [Disabling predefined rules](index.md#disable-predefined-analyzer-rules). -- [Overriding predefined rules](index.md#override-predefined-analyzer-rules). -- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](index.md#synthesize-a-custom-configuration). +- [Disabling predefined rules](#disable-predefined-analyzer-rules). +- [Overriding predefined rules](#override-predefined-analyzer-rules). +- Modifying the default behavior of the Secret Detection analyzer by [synthesizing and passing a custom configuration](#synthesize-a-custom-configuration). Customization allows replacing the default secret detection rules with rules that you define. @@ -334,7 +334,7 @@ To create a custom configuration, you can use passthrough chains. ``` Passthroughs can also be chained to build more complex configurations. -For more details, see [SAST Customize ruleset section](../sast/index.md#customize-rulesets). +For more details, see [SAST Customize ruleset section](../sast/customize_rulesets.md). ### Logging level diff --git a/doc/user/application_security/vulnerabilities/index.md b/doc/user/application_security/vulnerabilities/index.md index f0ac01000ef..ad397c3fe04 100644 --- a/doc/user/application_security/vulnerabilities/index.md +++ b/doc/user/application_security/vulnerabilities/index.md @@ -152,8 +152,8 @@ includes a **Resolve with merge request** option. The following scanners are supported by this feature: - [Dependency Scanning](../dependency_scanning/index.md). - Automatic Patch creation is only available for Node.js projects managed with - `yarn` when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is disabled. + Automatic patch creation is only available for Node.js projects managed with + `yarn`. Also, Automatic patch creation is only supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is disabled. - [Container Scanning](../container_scanning/index.md). To resolve a vulnerability, you can either: diff --git a/doc/user/application_security/vulnerability_report/pipeline.md b/doc/user/application_security/vulnerability_report/pipeline.md index 14c13f74a5e..32916f4c9c7 100644 --- a/doc/user/application_security/vulnerability_report/pipeline.md +++ b/doc/user/application_security/vulnerability_report/pipeline.md @@ -60,14 +60,14 @@ To download a security scan output: ## Scan results This shows a list of the combined results for all security report artifacts. The filters work like the -[Vulnerability Report filters](index.md#vulnerability-report-filters), but they are limited to **Severity** and **Tool**, with +[Vulnerability Report filters](index.md#vulnerability-report-filters), but they are limited to **Severity** and **Tool**, with the addition of a **Hide dismissed** toggle. -When you review the vulnerability findings reported in the pipeline, you can select one or more entries for dismissal, +When you review the vulnerability findings reported in the pipeline, you can select one or more entries for dismissal, similar to [Dismissing a vulnerability](index.md#dismissing-a-vulnerability) in the Vulnerability Report. When you merge the branch corresponding to the pipeline into the default branch, all reported findings are combined into -the [Vulnerability Report](index.md). Scan results in pipelines executed on the default branch are +the [Vulnerability Report](index.md). Scan results in pipelines executed on the default branch are incorporated once the pipeline finishes. | Existing vulnerability status | Dismissed in pipeline? | New vulnerability status | diff --git a/doc/user/asciidoc.md b/doc/user/asciidoc.md index b55a55eebe5..8de777672ff 100644 --- a/doc/user/asciidoc.md +++ b/doc/user/asciidoc.md @@ -477,8 +477,8 @@ digraph G { #### PlantUML -To make PlantUML available in GitLab, a GitLab administrator needs to enable it first. -Read more in [PlantUML & GitLab](../administration/integration/plantuml.md). +PlantUML integration is enabled on GitLab.com. To make PlantUML available in self-managed +installation of GitLab, a GitLab administrator [must enable it](../administration/integration/plantuml.md). After PlantUML is enabled, enter your text in a `plantuml` block: diff --git a/doc/user/clusters/agent/ci_cd_workflow.md b/doc/user/clusters/agent/ci_cd_workflow.md index dce02a72300..16b92eb92a3 100644 --- a/doc/user/clusters/agent/ci_cd_workflow.md +++ b/doc/user/clusters/agent/ci_cd_workflow.md @@ -25,7 +25,7 @@ To ensure access to your cluster is safe: - Each agent has a separate context (`kubecontext`). - Only the project where the agent is configured, and any additional projects you authorize, can access the agent in your cluster. -You do not need to have a runner in the cluster with the agent. +The CI/CD workflow requires runners to be registered with GitLab, but these runners do not have to be in the cluster where the agent is. ## GitLab CI/CD workflow steps @@ -127,8 +127,8 @@ Run `kubectl config get-contexts`. ### Environments with both certificate-based and agent-based connections -When you deploy to an environment that has both a [certificate-based -cluster](../../infrastructure/clusters/index.md) (deprecated) and an agent connection: +When you deploy to an environment that has both a +[certificate-based cluster](../../infrastructure/clusters/index.md) (deprecated) and an agent connection: - The certificate-based cluster's context is called `gitlab-deploy`. This context is always selected by default. diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index 73a35ffbc64..4978b56917b 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -4,10 +4,11 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Using GitOps with a Kubernetes cluster **(PREMIUM)** +# Using GitOps with a Kubernetes cluster **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/346567) from GitLab Premium to GitLab Free in 15.3. With GitOps, you can manage containerized clusters and applications from a Git repository that: diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index 9282ac7fb40..4b0d8b77493 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -17,7 +17,7 @@ To connect a Kubernetes cluster to GitLab, you must install an agent in your clu Before you can install the agent in your cluster, you need: - An existing Kubernetes cluster. If you don't have a cluster, you can create one on a cloud provider, like: - - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/quickstart) + - [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine/docs/deploy-app-cluster) - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) - On self-managed GitLab instances, a GitLab administrator must set up the diff --git a/doc/user/clusters/agent/vulnerabilities.md b/doc/user/clusters/agent/vulnerabilities.md index 3b80a7a0f81..5afe3ccec2b 100644 --- a/doc/user/clusters/agent/vulnerabilities.md +++ b/doc/user/clusters/agent/vulnerabilities.md @@ -16,6 +16,9 @@ You can also configure your agent so the vulnerabilities are displayed with othe You can use operational container scanning to scan container images in your cluster for security vulnerabilities. +NOTE: +In GitLab 15.0 and later, you do not need to install Starboard operator in the Kubernetes cluster. + To begin scanning all resources in your cluster, add a `starboard` configuration block to your agent configuration with a `cadence` field containing a CRON expression for when the scans will be run. diff --git a/doc/user/clusters/cost_management.md b/doc/user/clusters/cost_management.md index 47dc9c42797..3dcec32b90c 100644 --- a/doc/user/clusters/cost_management.md +++ b/doc/user/clusters/cost_management.md @@ -61,7 +61,7 @@ this dashboard. You can customize the cost dashboard by editing the `.gitlab/dashboards/default_costs.yml` file or creating similar dashboard configuration files. To learn more, read about -[customizing dashboards in our documentation](/ee/operations/metrics/dashboards/). +[customizing dashboards in our documentation](../../operations/metrics/dashboards/index.md). #### Available metrics diff --git a/doc/user/clusters/environments.md b/doc/user/clusters/environments.md index b7732a7abf8..cf71729b517 100644 --- a/doc/user/clusters/environments.md +++ b/doc/user/clusters/environments.md @@ -33,8 +33,8 @@ With cluster environments, you can gain insight into:  -Access to cluster environments is restricted to [group maintainers and -owners](../permissions.md#group-members-permissions) +Access to cluster environments is restricted to +[group maintainers and owners](../permissions.md#group-members-permissions) ## Usage diff --git a/doc/user/clusters/management_project.md b/doc/user/clusters/management_project.md index d59edb1e2c2..361276194b0 100644 --- a/doc/user/clusters/management_project.md +++ b/doc/user/clusters/management_project.md @@ -81,8 +81,7 @@ configure cluster: ### Setting the environment scope -[Environment -scopes](../project/clusters/multiple_kubernetes_clusters.md#setting-the-environment-scope) +[Environment scopes](../project/clusters/multiple_kubernetes_clusters.md#setting-the-environment-scope) are usable when associating multiple clusters to the same management project. diff --git a/doc/user/clusters/management_project_template.md b/doc/user/clusters/management_project_template.md index 7ab77c67bcc..4b00784a7ae 100644 --- a/doc/user/clusters/management_project_template.md +++ b/doc/user/clusters/management_project_template.md @@ -15,6 +15,9 @@ to create a project. The project includes cluster applications that integrate wi and extend GitLab functionality. You can use the pattern shown in the project to extend your custom cluster applications. +NOTE: +The project template works on GitLab.com without modifications. If you're on a self-managed instance, you must modify the `.gitlab-ci.yml` file. + ## Use one project for the agent and your manifests If you **have not yet** used the agent to connect your cluster with GitLab: @@ -47,10 +50,7 @@ To create a project from the cluster management project template: 1. From the list of templates, next to **GitLab Cluster Management**, select **Use template**. 1. Enter the project details. 1. Select **Create project**. - -If you use self-managed GitLab, your instance might not include the latest version of the template. -In that case, select **Import project**, **Repository by URL** and for the **Git repository URL**, enter -`https://gitlab.com/gitlab-org/project-templates/cluster-management.git`. +1. In the new project, [configure the files](#configure-the-project) as needed. ## Configure the project @@ -73,6 +73,11 @@ The base image used in the pipeline is built by the [cluster-applications](https://gitlab.com/gitlab-org/cluster-integration/cluster-applications) project. This image contains a set of Bash utility scripts to support [Helm v3 releases](https://helm.sh/docs/intro/using_helm/#three-big-concepts). +If you are on a self-managed instance of GitLab, you must modify the `.gitlab-ci.yml` file. +Specifically, the section that starts with the comment `Automatic package upgrades` will not +work on a self-managed instance, because the `include` refers to a GitLab.com project. +If you remove everything below this comment, the pipeline will succeed. + ### The main `helmfile.yml` file The template contains a [Helmfile](https://github.com/roboll/helmfile) you can use to manage diff --git a/doc/user/compliance/compliance_report/index.md b/doc/user/compliance/compliance_report/index.md index f547e5f146f..4cd2705e917 100644 --- a/doc/user/compliance/compliance_report/index.md +++ b/doc/user/compliance/compliance_report/index.md @@ -92,30 +92,49 @@ Our criteria for the separation of duties is as follows: ## Chain of Custody report -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213364) in GitLab 13.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213364) in GitLab 13.3. +> - Chain of Custody reports sent using email [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342594) in GitLab 15.3 with a flag named `async_chain_of_custody_report`. Disabled by default. + +FLAG: +On self-managed GitLab, by default sending Chain of Custody reports through email is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `async_chain_of_custody_report`. On GitLab.com, this feature is not available. The Chain of Custody report allows customers to export a list of merge commits within the group. The data provides a comprehensive view with respect to merge commits. It includes the merge commit SHA, merge request author, merge request ID, merge user, date merged, pipeline ID, group name, project name, and merge request approvers. Depending on the merge strategy, the merge commit SHA can be a merge commit, squash commit, or a diff head commit. -To download the Chain of Custody report: +To generate the Chain of Custody report: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Security & Compliance > Compliance report**. 1. Select **List of all merge commits**. -### Commit-specific Chain of Custody Report +The Chain of Custody report is either: + +- Available for download. +- Sent through email. Requires GitLab 15.3 and later with `async_chain_of_custody_report` feature flag enabled. + +### Commit-specific Chain of Custody report > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267629) in GitLab 13.6. -You can generate a commit-specific Chain of Custody report for a given commit SHA. +Authenticated group owners can generate a commit-specific Chain of Custody report for a given commit SHA, either: -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Compliance report**. -1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{chevron-lg-down}**). -1. Enter the merge commit SHA, and then select **Export commit custody report**. - SHA and then select **Export commit custody report**. +- Using the GitLab UI: + + 1. On the top bar, select **Menu > Groups** and find your group. + 1. On the left sidebar, select **Security & Compliance > Compliance report**. + 1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{chevron-lg-down}**). + 1. Enter the merge commit SHA, and then select **Export commit custody report**. + SHA and then select **Export commit custody report**. + +The Chain of Custody report is either: + +- Available for download. +- Sent through email. Requires GitLab 15.3 and later with `async_chain_of_custody_report` feature flag enabled. + +- Using a direct link: `https://gitlab.com/groups/<group-name>/-/security/merge_commit_reports.csv?commit_sha={optional_commit_sha}`, passing in an optional value to the + `commit_sha` query parameter. NOTE: The Chain of Custody report download is a CSV file, with a maximum size of 15 MB. diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 8c57220068b..1c9f9e85ab7 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -71,13 +71,11 @@ Gradle 1.x projects are not supported. The minimum supported version of Maven is |------------|----------------------------------------------------------------------------------------------|-------| | JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) (7 and earlier) | | | Go | [Godep](https://github.com/tools/godep) ([deprecated](../../../update/deprecations.md#godep-support-in-license-compliance)), [go mod](https://github.com/golang/go/wiki/Modules) | | -| Java | [Gradle](https://gradle.org/) <sup>1</sup>, [Maven](https://maven.apache.org/) | | +| Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) | | | .NET | [NuGet](https://www.nuget.org/) | The .NET Framework is supported via the [mono project](https://www.mono-project.com/). There are, however, some limitations. The scanner doesn't support Windows-specific dependencies and doesn't report dependencies of your project's listed dependencies. Also, the scanner always marks detected licenses for all dependencies as `unknown`. | | Python | [pip](https://pip.pypa.io/en/stable/) | Python is supported through [requirements.txt](https://pip.pypa.io/en/stable/user_guide/#requirements-files) and [Pipfile.lock](https://github.com/pypa/pipfile#pipfilelock). | | Ruby | [gem](https://rubygems.org/) | | -1. Gradle 7 and later is not supported as dependencies are not discovered when included with the `implementation` directive. Please see [GitLab#341222](https://gitlab.com/gitlab-org/gitlab/-/issues/341222) for more details. - ### Experimental support The following languages and package managers are [supported experimentally](https://github.com/pivotal/LicenseFinder#experimental-project-types). @@ -468,7 +466,7 @@ package manager. For a comprehensive list, consult [the Conan documentation](htt The default [Conan](https://conan.io/) configuration sets [`CONAN_LOGIN_USERNAME`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) to `ci_user`, and binds [`CONAN_PASSWORD`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-password-conan-password-remote-name) to the [`CI_JOB_TOKEN`](../../../ci/variables/predefined_variables.md) -for the running job. This allows Conan projects to fetch packages from a [GitLab Conan Repository](../../packages/conan_repository/#fetch-conan-package-information-from-the-package-registry) +for the running job. This allows Conan projects to fetch packages from a [GitLab Conan Repository](../../packages/conan_repository/index.md#fetch-conan-package-information-from-the-package-registry) if a GitLab remote is specified in the `.conan/remotes.json` file. To override the default credentials specify a [`CONAN_LOGIN_USERNAME_{REMOTE_NAME}`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name) diff --git a/doc/user/crm/index.md b/doc/user/crm/index.md index a2cfdf61a8d..e71a983ccfd 100644 --- a/doc/user/crm/index.md +++ b/doc/user/crm/index.md @@ -169,7 +169,7 @@ You can also add, remove, or replace issue contacts using the [GraphQL](../../api/graphql/reference/index.md#mutationissuesetcrmcontacts) API. -## Autocomplete contacts **(FREE SELF)** +## Autocomplete contacts > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2256) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `contacts_autocomplete`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/352123) in GitLab 15.0. diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 1f34d182718..3fb0be6480c 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -47,7 +47,19 @@ in a different color. Avoid mentioning `@all` in issues and merge requests, because it sends an email notification to all the members of that project's group. This might be interpreted as spam. Notifications and mentions can be disabled in -[a group's settings](../group/index.md#disable-email-notifications). +[a group's settings](../group/manage.md#disable-email-notifications). + +### Mention a group in an issue or merge request + +When you mention a group in a comment, every member of the group gets a to-do item +added to their To-do list. + +1. Open the MR or issue. +1. In a comment, type `@` followed by the user, group, or subgroup namespace. + For example, `@alex`, `@alex-team`, or `@alex-team/marketing`. +1. Select **Comment**. + +A to-do item is created for all the group and subgroup members. ## Add a comment to a merge request diff @@ -347,3 +359,19 @@ with a new push. Threads are now resolved if a push makes a diff section outdated. Threads on lines that don't change and top-level resolvable threads are not resolved. + +## Display paginated merge request discussions + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340172) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `paginated_mr_discussions`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/364497) in GitLab 15.2. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/364497) in GitLab 15.3. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature +per project or for your entire instance, ask an administrator to +[disable the feature flag](../../administration/feature_flags.md) named `paginated_mr_discussions`. +On GitLab.com, this feature is available. + +A merge request can have many discussions. Loading them all in a single request +can be slow. To improve the performance of loading discussions, they are split into multiple +pages, loading sequentially. diff --git a/doc/user/feature_highlight.md b/doc/user/feature_highlight.md index e5f0369a0f6..ef96d2524a6 100644 --- a/doc/user/feature_highlight.md +++ b/doc/user/feature_highlight.md @@ -1,15 +1,11 @@ --- -stage: none -group: unassigned -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'index.md' +remove_date: '2022-10-29' --- -# Feature highlight +This document was moved to [another location](index.md). -Feature highlights are represented by a pulsing blue dot. Hovering over the dot -displays more information. They're used to emphasize certain features and -highlight helpful information to the user. - -You can dismiss any feature highlight permanently by clicking the **Got it** link -at the bottom of the information window. You cannot restore a feature highlight -after you dismiss it. +<!-- This redirect file can be deleted after <2022-10-29>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (for example, link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/free_user_limit.md b/doc/user/free_user_limit.md index b848128b160..a62b9c9e363 100644 --- a/doc/user/free_user_limit.md +++ b/doc/user/free_user_limit.md @@ -6,8 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Free user limit **(FREE SAAS)** -From September 15, 2022, namespaces in GitLab.com on the Free tier -will be limited to five (5) members per [namespace](group/index.md#namespaces). +From October 19, 2022, namespaces in GitLab.com on the Free tier +will be limited to five (5) members per [namespace](namespace/index.md). This limit applies to top-level groups and personal namespaces. In a personal namespace, the limit applies across all projects in your personal diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 529b81e2645..53e459f7a09 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -96,7 +96,7 @@ Projects are permanently deleted after a seven-day delay. If you are on: -- Premium tier and above, you can disable this by changing the [group setting](../group/index.md#enable-delayed-project-deletion). +- Premium tier and above, you can disable this by changing the [group setting](../group/manage.md#enable-delayed-project-deletion). - Free tier, you cannot disable this setting or restore projects. ## Inactive project deletion @@ -322,7 +322,7 @@ The list of GitLab.com specific settings (and their defaults) is as follows: Some of these settings are in the process being adjusted. For example, the value for `shared_buffers` is quite high, and we are -[considering adjusting it](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/4985). +[considering adjusting it](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4985). ## Puma @@ -336,8 +336,8 @@ documentation. When a request is rate limited, GitLab responds with a `429` status code. The client should wait before attempting the request again. There -are also informational headers with this response detailed in [rate -limiting responses](#rate-limiting-responses). +are also informational headers with this response detailed in +[rate limiting responses](#rate-limiting-responses). The following table describes the rate limits for GitLab.com, both before and after the limits change in January, 2021: @@ -358,9 +358,9 @@ after the limits change in January, 2021: | **Pipeline creation** requests (for a given **project, user, and commit**) | | **25** requests per minute | | **Alert integration endpoint** requests (for a given **project**) | | **3600** requests per hour | -More details are available on the rate limits for [protected -paths](#protected-paths-throttle) and [raw -endpoints](../../user/admin_area/settings/rate_limits_on_raw_endpoints.md). +More details are available on the rate limits for +[protected paths](#protected-paths-throttle) and +[raw endpoints](../../user/admin_area/settings/rate_limits_on_raw_endpoints.md). GitLab can rate-limit requests at several layers. The rate limits listed here are configured in the application. These limits are the most @@ -398,7 +398,7 @@ following section. If you receive a `403 Forbidden` error for all requests to GitLab.com, check for any automated processes that may be triggering a block. For -assistance, contact [GitLab Support](https://support.gitlab.com/hc/en-us) +assistance, contact [GitLab Support](https://support.gitlab.com) with details, such as the affected IP address. #### Git and container registry failed authentication ban @@ -424,13 +424,9 @@ For performance reasons, if a query returns more than 10,000 records, [GitLab ex ### Visibility settings -If created before GitLab 12.2 (July 2019), these items have the +Projects, groups, and snippets have the [Internal visibility](../public_access.md#internal-projects-and-groups) -setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/12388): - -- Projects -- Groups -- Snippets +setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/12388). ### SSH maximum number of connections diff --git a/doc/user/group/access_and_permissions.md b/doc/user/group/access_and_permissions.md new file mode 100644 index 00000000000..c469d6c2f6d --- /dev/null +++ b/doc/user/group/access_and_permissions.md @@ -0,0 +1,260 @@ +--- +stage: Manage +group: Workspace +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Group access and permissions + +Configure your groups to control group permissions and access. + +## Group push rules **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34370) in GitLab 12.8. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/224129) in GitLab 13.4. + +Group push rules allow group maintainers to set +[push rules](../project/repository/push_rules.md) for newly created projects in the specific group. + +To configure push rules for a group: + +1. Go to the groups's **Push Rules** page. +1. Select the settings you want. +1. Select **Save Push Rules**. + +The group's new subgroups have push rules set for them based on either: + +- The closest parent group with push rules defined. +- Push rules set at the instance level, if no parent groups have push rules defined. + +## Restrict group access by IP address **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1985) in GitLab 12.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215410) from GitLab Ultimate to GitLab Premium in 13.1. + +To ensure only people from your organization can access particular +resources, you can restrict access to groups by IP address. This group-level setting +applies to: + +- The GitLab UI, including subgroups, projects, and issues. +- [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API. + +### Security implications + +You should consider some security implications before configuring IP address restrictions. + +- Restricting HTTP traffic on GitLab.com with IP address restrictions causes SSH requests (including Git operations over + SSH) to fail. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/271673). +- Administrators and group owners can access group settings from any IP address, regardless of IP restriction. However: + - Groups owners cannot access projects belonging to the group when accessing from a disallowed IP address. + - Administrators can access projects belonging to the group when accessing from a disallowed IP address. + Access to projects includes cloning code from them. + - Users can still see group and project names and hierarchies. Only the following are restricted: + - [Groups](../../api/groups.md), including all [group resources](../../api/api_resources.md#group-resources). + - [Project](../../api/projects.md), including all [project resources](../../api/api_resources.md#project-resources). +- When you register a runner, it is not bound by the IP restrictions. When the runner requests a new job or an update to + a job's state, it is also not bound by the IP restrictions. But when the running CI/CD job sends Git requests from a + restricted IP address, the IP restriction prevents code from being cloned. +- Users may still see some events from the IP restricted groups and projects on their dashboard. Activity may include + push, merge, issue, or comment events. + +### Restrict group access by IP address + +To restrict group access by IP address: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. In the **Allow access to the following IP addresses** field, enter IPv4 or IPv6 address ranges in CIDR notation. +1. Select **Save changes**. + +In self-managed installations of GitLab 15.1 and later, you can also configure +[globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges) +at the group level. + +## Restrict group access by domain **(PREMIUM)** + +> - Support for specifying multiple email domains [added](https://gitlab.com/gitlab-org/gitlab/-/issues/33143) in GitLab 13.1. +> - Support for restricting access to projects in the group [added](https://gitlab.com/gitlab-org/gitlab/-/issues/14004) in GitLab 14.1.2. +> - Support for restricting group memberships to groups with a subset of the allowed email domains [added](https://gitlab.com/gitlab-org/gitlab/-/issues/354791) in GitLab 15.1.1 + +You can prevent users with email addresses in specific domains from being added to a group and its projects. + +To restrict group access by domain: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. In the **Restrict membership by email** field, enter the domain names. +1. Select **Save changes**. + +Any time you attempt to add a new user, the user's [primary email](../profile/index.md#change-your-primary-email) is compared against this list. +Only users with a [primary email](../profile/index.md#change-your-primary-email) that matches any of the configured email domain restrictions +can be added to the group. + +The most popular public email domains cannot be restricted, such as: + +- `gmail.com`, `yahoo.com`, `aol.com`, `icloud.com` +- `hotmail.com`, `hotmail.co.uk`, `hotmail.fr` +- `msn.com`, `live.com`, `outlook.com` + +When you share a group, both the source and target namespaces must allow the domains of the members' email addresses. + +NOTE: +Removing a domain from the **Restrict membership by email** list does not remove the users with this email domain from the groups and projects under this group. +Also, if you share a group or project with another group, the target group can add more email domains to its list that are not in the list of the source group. +Hence, this feature does not ensure that the current members always conform to the **Restrict membership by email** list. + +## Prevent group sharing outside the group hierarchy + +You can configure a top-level group so its subgroups and projects +cannot invite other groups outside of the top-level group's hierarchy. +This option is only available for top-level groups. + +For example, in the following group and project hierarchy: + +- **Animals > Dogs > Dog Project** +- **Animals > Cats** +- **Plants > Trees** + +If you prevent group sharing outside the hierarchy for the **Animals** group: + +- **Dogs** can invite the group **Cats**. +- **Dogs** cannot invite the group **Trees**. +- **Dog Project** can invite the group **Cats**. +- **Dog Project** cannot invite the group **Trees**. + +To prevent sharing outside of the group's hierarchy: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Permissions and group features**. +1. Select **Prevent members from sending invitations to groups outside of `<group_name>` and its subgroups**. +1. Select **Save changes**. + +## Prevent a project from being shared with groups + +Prevent projects in a group from +[sharing a project with another group](../project/members/share_project_with_groups.md) +to enable tighter control over project access. + +To prevent a project from being shared with other groups: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Select **Prevent sharing a project in `<group_name>` with other groups**. +1. Select **Save changes**. + +This setting applies to all subgroups unless overridden by a group owner. Groups already +added to a project lose access when the setting is enabled. + +## Prevent users from requesting access to a group + +As a group owner, you can prevent non-members from requesting access to +your group. + +1. On the top bar, select **Menu > Groups**. +1. Select **Your Groups**. +1. Find the group and select it. +1. From the left menu, select **Settings > General**. +1. Expand the **Permissions and group features** section. +1. Clear the **Allow users to request access** checkbox. +1. Select **Save changes**. + +## Prevent project forking outside group **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216987) in GitLab 13.3. + +By default, projects in a group can be forked. +Optionally, on [GitLab Premium](https://about.gitlab.com/pricing/) or higher tiers, +you can prevent the projects in a group from being forked outside of the current top-level group. + +This setting will be removed from the SAML setting page, and migrated to the +group settings page. In the interim period, both of these settings are taken into consideration. +If even one is set to `true`, then the group does not allow outside forks. + +To prevent projects from being forked outside the group: + +1. Go to the top-level group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Check **Prevent project forking outside current group**. +1. Select **Save changes**. + +Existing forks are not removed. + +## Prevent members from being added to projects in a group **(PREMIUM)** + +As a group owner, you can prevent any new project membership for all +projects in a group, allowing tighter control over project membership. + +For example, if you want to lock the group for an [Audit Event](../../administration/audit_events.md), +you can guarantee that project membership cannot be modified during the audit. + +You can still invite groups or to add members to groups, implicitly giving members access to projects in the **locked** group. + +The setting does not cascade. Projects in subgroups observe the subgroup configuration, ignoring the parent group. + +To prevent members from being added to projects in a group: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Under **Membership**, select **Prevent adding new members to projects within this group**. +1. Select **Save changes**. + +All users who previously had permissions can no longer add members to a group. +API requests to add a new user to a project are not possible. + +## Manage group memberships via LDAP **(PREMIUM SELF)** + +Group syncing allows LDAP groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing, edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that are associated with GitLab groups. + +Group links can be created by using either a CN or a filter. To create these group links, go to the group's **Settings > LDAP Synchronization** page. After configuring the link, it may take more than an hour for the users to sync with the GitLab group. + +For more information on the administration of LDAP and group sync, refer to the [main LDAP documentation](../../administration/auth/ldap/ldap_synchronization.md#group-sync). + +NOTE: +When you add LDAP synchronization, if an LDAP user is a group member and they are not part of the LDAP group, they are removed from the group. + +### Create group links via CN **(PREMIUM SELF)** + +To create group links via CN: + +<!-- vale gitlab.Spelling = NO --> + +1. Select the **LDAP Server** for the link. +1. As the **Sync method**, select `LDAP Group cn`. +1. In the **LDAP Group cn** field, begin typing the CN of the group. There is a dropdown list with matching CNs in the configured `group_base`. Select your CN from this list. +1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. +1. Select **Add Synchronization**. + +<!-- vale gitlab.Spelling = YES --> + +### Create group links via filter **(PREMIUM SELF)** + +To create group links via filter: + +1. Select the **LDAP Server** for the link. +1. As the **Sync method**, select `LDAP user filter`. +1. Input your filter in the **LDAP User filter** box. Follow the [documentation on user filters](../../administration/auth/ldap/index.md#set-up-ldap-user-filter). +1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. +1. Select **Add Synchronization**. + +### Override user permissions **(PREMIUM SELF)** + +LDAP user permissions can be manually overridden by an administrator. To override a user's permissions: + +1. Go to your group's **Group information > Members** page. +1. In the row for the user you are editing, select the pencil (**{pencil}**) icon. +1. Select **Edit permissions** in the modal. + +Now you can edit the user's permissions from the **Members** page. + +## Troubleshooting + +### Verify if access is blocked by IP restriction + +If a user sees a 404 when they would normally expect access, and the problem is limited to a specific group, search the `auth.log` rails log for one or more of the following: + +- `json.message`: `'Attempting to access IP restricted group'` +- `json.allowed`: `false` + +In viewing the log entries, compare the `remote.ip` with the list of +[allowed IPs](#restrict-group-access-by-ip-address) for the group. diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index 6755bf9ffb9..43587dd54ef 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -30,7 +30,7 @@ To set up custom project templates in a group, add the subgroup that contains th project templates to the group settings: 1. In the group, create a [subgroup](subgroups/index.md). -1. [Add projects to the new subgroup](index.md#add-projects-to-a-group) as your templates. +1. [Add projects to the new subgroup](manage.md#add-projects-to-a-group) as your templates. 1. In the left menu for the group, select **Settings > General**. 1. Expand **Custom project templates** and select the subgroup. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index 56d1569c908..c7d92af8ee8 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -18,17 +18,9 @@ You can use groups to communicate with all of the members of the group at once. For larger organizations, you can also create [subgroups](subgroups/index.md). -## View groups +For more information about creating and managing your groups, see [Manage groups](manage.md). -To view groups: - -1. On the top bar, select **Menu > Groups**. -1. Select **Your Groups**. All groups you are a member of are displayed. -1. To view a list of public groups, select **Explore public groups**. - -You can also view groups by namespace. - -### Group visibility +## Group visibility Like projects, a group can be configured to limit the visibility of it to: @@ -43,816 +35,6 @@ empty for anonymous users. The group page has a visibility level icon. Administrator users cannot create a subgroup or project with a higher visibility level than that of the immediate parent group. -### Namespaces - -In GitLab, a *namespace* organizes related projects together. -GitLab has two types of namespaces: - -- A *personal* namespace, which is based on your username. Projects under a personal namespace must be configured one at a time. -- A *group* or *subgroup* namespace. In these namespaces, you can manage multiple projects at once. - -To determine whether you're viewing a group or personal namespace, you can view the URL. For example: - -| Namespace for | URL | Namespace | -| ------------- | --- | --------- | -| A user named `alex`. | `https://gitlab.example.com/alex` | `alex` | -| A group named `alex-team`. | `https://gitlab.example.com/alex-team` | `alex-team` | -| A group named `alex-team` with a subgroup named `marketing`. | `https://gitlab.example.com/alex-team/marketing` | `alex-team/marketing` | - -## Create a group - -To create a group: - -1. On the top bar, either: - - Select **Menu > Groups**, and on the right, select **Create group**. - - To the left of the search box, select the plus sign and then **New group**. -1. Select **Create group**. -1. Enter a name for the group in **Group name**. For a list of words that cannot be used as group names, see - [reserved names](../reserved_names.md). -1. Enter a path for the group in **Group URL**, which is used for the [namespace](#namespaces). -1. Choose the [visibility level](../public_access.md). -1. Personalize your GitLab experience by answering the following questions: - - What is your role? - - Who will be using this group? - - What will you use this group for? -1. Invite GitLab members or other users to join the group. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For details about groups, watch [GitLab Namespaces (users, groups and subgroups)](https://youtu.be/r0sJgjR2f5A). - -## Add users to a group - -You can give a user access to all projects in a group. - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Group information > Members**. -1. Select **Invite members**. -1. Fill in the fields. - - The role applies to all projects in the group. [Learn more about permissions](../permissions.md). - - On the **Access expiration date**, the user can no longer access projects in the group. -1. Select **Invite**. - -Members that are not automatically added are displayed on the **Invited** tab. -Users can be on this tab because they: - -- Have not yet accepted the invitation. -- Are waiting for [approval from an administrator](../admin_area/moderate_users.md). -- [Exceed the group user cap](#user-cap-for-groups). - -## Request access to a group - -As a user, you can request to be a member of a group, if an administrator allows it. - -1. On the top bar, select **Menu > Groups** and find your group. -1. Under the group name, select **Request Access**. - -As many as ten of the most-recently-active group owners receive an email with your request. -Any group owner can approve or decline the request. - -If you change your mind before your request is approved, select -**Withdraw Access Request**. - -## Prevent users from requesting access to a group - -As a group owner, you can prevent non-members from requesting access to -your group. - -1. On the top bar, select **Menu > Groups**. -1. Select **Your Groups**. -1. Find the group and select it. -1. From the left menu, select **Settings > General**. -1. Expand the **Permissions and group features** section. -1. Clear the **Allow users to request access** checkbox. -1. Select **Save changes**. - -## Change the owner of a group - -You can change the owner of a group. Each group must always have at least one -member with the Owner role. - -- As an administrator: - 1. Go to the group and from the left menu, select **Group information > Members**. - 1. Give a different member the **Owner** role. - 1. Refresh the page. You can now remove the **Owner** role from the original owner. -- As the current group's owner: - 1. Go to the group and from the left menu, select **Group information > Members**. - 1. Give a different member the **Owner** role. - 1. Have the new owner sign in and remove the **Owner** role from you. - -## Remove a member from the group - -Prerequisites: - -- You must have the Owner role. -- The member must have direct membership in the group. If - membership is inherited from a parent group, then the member can be removed - from the parent group only. - -To remove a member from a group: - -1. Go to the group. -1. From the left menu, select **Group information > Members**. -1. Next to the member you want to remove, select **Delete**. -1. Optional. On the **Remove member** confirmation box, select the - **Also unassign this user from linked issues and merge requests** checkbox. -1. Select **Remove member**. - -## Filter and sort members in a group - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21727) in GitLab 12.6. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/228675) in GitLab 13.7. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289911) in GitLab 13.8. - -To find members in a group, you can sort, filter, or search. - -### Filter a group - -Filter a group to find members. By default, all members in the group and subgroups are displayed. - -1. Go to the group and select **Group information > Members**. -1. Above the list of members, in the **Filter members** box, enter filter criteria. - - To view members in the group only, select **Membership = Direct**. - - To view members of the group and its subgroups, select **Membership = Inherited**. - - To view members with two-factor authentication enabled or disabled, select **2FA = Enabled** or **Disabled**. - - [In GitLab 14.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/349887), to view GitLab users created by [SAML SSO](saml_sso/index.md) or [SCIM provisioning](saml_sso/scim_setup.md) select **Enterprise = true**. - -### Search a group - -You can search for members by name, username, or email. - -1. Go to the group and select **Group information > Members**. -1. Above the list of members, in the **Filter members** box, enter search criteria. -1. To the right of the **Filter members** box, select the magnifying glass (**{search}**). - -### Sort members in a group - -You can sort members by **Account**, **Access granted**, **Max role**, or **Last sign-in**. - -1. Go to the group and select **Group information > Members**. -1. Above the list of members, on the top right, from the **Account** list, select - the criteria to filter by. -1. To switch the sort between ascending and descending, to the right of the **Account** list, select the - arrow (**{sort-lowest}** or **{sort-highest}**). - -## Mention a group in an issue or merge request - -When you mention a group in a comment, every member of the group gets a to-do item -added to their To-do list. - -1. Open the MR or issue. -1. In a comment, type `@` followed by the user, group, or subgroup namespace. - For example, `@alex`, `@alex-team`, or `@alex-team/marketing`. -1. Select **Comment**. - -A to-do item is created for all the group and subgroup members. - -## Change the default branch protection of a group - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7583) in GitLab 12.9. -> - [Settings moved and renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/340403) in GitLab 14.9. - -By default, every group inherits the branch protection set at the global level. - -To change this setting for a specific group, see [group level default branch protection](../project/repository/branches/default.md#group-level-default-branch-protection). - -To change this setting globally, see [initial default branch protection](../project/repository/branches/default.md#instance-level-default-branch-protection). - -NOTE: -In [GitLab Premium or higher](https://about.gitlab.com/pricing/), GitLab administrators can choose to [disable group owners from updating the default branch protection](../project/repository/branches/default.md#prevent-overrides-of-default-branch-protection). - -## Add projects to a group - -There are two different ways to add a new project to a group: - -- Select a group, and then select **New project**. You can then continue [creating your project](../../user/project/working_with_projects.md#create-a-project). -- While you are creating a project, select a group from the dropdown list. - -  - -### Specify who can add projects to a group - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2534) in GitLab 10.5. -> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25975) from GitLab Premium to GitLab Free in 11.10. - -By default, users with at least the Developer role can create projects under a group. - -To change this setting for a specific group: - -1. On the top bar, select **Menu > Groups**. -1. Select **Your Groups**. -1. Find the group and select it. -1. From the left menu, select **Settings > General**. -1. Expand the **Permissions and group features** section. -1. Select the desired option in the **Roles allowed to create projects** dropdown list. -1. Select **Save changes**. - -To change this setting globally, see [Default project creation protection](../admin_area/settings/visibility_and_access_controls.md#define-which-roles-can-create-projects). - -## Group activity analytics **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207164) in GitLab 12.10 as a [Beta feature](../../policy/alpha-beta-support.md#beta-features). - -For a group, you can view how many merge requests, issues, and members were created in the last 90 days. - -These Group Activity Analytics can be enabled with the `group_activity_analytics` [feature flag](../../development/feature_flags/index.md#enabling-a-feature-flag-locally-in-development). - - - -Changes to [group wikis](../project/wiki/group.md) do not appear in group activity analytics. - -### View group activity - -You can view the most recent actions taken in a group, either in your browser or in an RSS feed: - -1. On the top bar, select **Menu > Groups**. -1. Select **Your Groups**. -1. Find the group and select it. -1. On the left sidebar, select **Group information > Activity**. - -To view the activity feed in Atom format, select the -**RSS** (**{rss}**) icon. - -## Share a group with another group - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18328) in GitLab 12.7. -> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 13.11 from a form to a modal window [with a flag](../feature_flags.md). Disabled by default. -> - Modal window [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 14.8. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) in GitLab 14.9. - [Feature flag `invite_members_group_modal`](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) removed. - -Similar to how you [share a project with a group](../project/members/share_project_with_groups.md), -you can share a group with another group. To invite a group, you must be a member of it. Members get direct access -to the shared group. This includes members who inherited group membership from a parent group. - -To share a given group, for example, `Frontend` with another group, for example, -`Engineering`: - -1. Go to the `Frontend` group. -1. On the left sidebar, select **Group information > Members**. -1. Select **Invite a group**. -1. In the **Select a group to invite** list, select `Engineering`. -1. Select a [role](../permissions.md) as maximum access level. -1. Select **Invite**. - -After sharing the `Frontend` group with the `Engineering` group: - -- The **Groups** tab lists the `Engineering` group. -- The **Groups** tab lists a group regardless of whether it is a public or private group. -- All members of the `Engineering` group have access to the `Frontend` group. The same access levels of the members apply up to the maximum access level selected when sharing the group. - -## Manage group memberships via LDAP **(PREMIUM SELF)** - -Group syncing allows LDAP groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing, edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that are associated with GitLab groups. - -Group links can be created by using either a CN or a filter. To create these group links, go to the group's **Settings > LDAP Synchronization** page. After configuring the link, it may take more than an hour for the users to sync with the GitLab group. - -For more information on the administration of LDAP and group sync, refer to the [main LDAP documentation](../../administration/auth/ldap/ldap_synchronization.md#group-sync). - -NOTE: -When you add LDAP synchronization, if an LDAP user is a group member and they are not part of the LDAP group, they are removed from the group. - -### Create group links via CN **(PREMIUM SELF)** - -To create group links via CN: - -<!-- vale gitlab.Spelling = NO --> - -1. Select the **LDAP Server** for the link. -1. As the **Sync method**, select `LDAP Group cn`. -1. In the **LDAP Group cn** field, begin typing the CN of the group. There is a dropdown list with matching CNs in the configured `group_base`. Select your CN from this list. -1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. -1. Select **Add Synchronization**. - -<!-- vale gitlab.Spelling = YES --> - -### Create group links via filter **(PREMIUM SELF)** - -To create group links via filter: - -1. Select the **LDAP Server** for the link. -1. As the **Sync method**, select `LDAP user filter`. -1. Input your filter in the **LDAP User filter** box. Follow the [documentation on user filters](../../administration/auth/ldap/index.md#set-up-ldap-user-filter). -1. In the **LDAP Access** section, select the [permission level](../permissions.md) for users synced in this group. -1. Select **Add Synchronization**. - -### Override user permissions **(PREMIUM SELF)** - -LDAP user permissions can be manually overridden by an administrator. To override a user's permissions: - -1. Go to your group's **Group information > Members** page. -1. In the row for the user you are editing, select the pencil (**{pencil}**) icon. -1. Select **Edit permissions** in the modal. - -Now you can edit the user's permissions from the **Members** page. - -## Transfer a group - -You can transfer groups in the following ways: - -- Transfer a subgroup to a new parent group. -- Convert a top-level group into a subgroup by transferring it to the desired group. -- Convert a subgroup into a top-level group by transferring it out of its current group. - -When transferring groups, note: - -- Changing a group's parent can have unintended side effects. See [what happens when a repository path changes](../project/repository/index.md#what-happens-when-a-repository-path-changes). -- You can only transfer groups to groups you manage. -- You must update your local repositories to point to the new location. -- If the immediate parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects change to match the new parent group's visibility. -- Only explicit group membership is transferred, not inherited membership. If the group's owners have only inherited membership, this leaves the group without an owner. In this case, the user transferring the group becomes the group's owner. -- Transfers fail if [packages](../packages/index.md) exist in any of the projects in the group, or in any of its subgroups. - -## Change a group's path - -Changing a group's path (group URL) can have unintended side effects. Read -[how redirects behave](../project/repository/index.md#what-happens-when-a-repository-path-changes) -before you proceed. - -If you are changing the path so it can be claimed by another group or user, -you must rename the group too. Both names and paths must -be unique. - -To retain ownership of the original namespace and protect the URL redirects, -create a new group and transfer projects to it instead. - -To change your group path (group URL): - -1. Go to your group's **Settings > General** page. -1. Expand the **Advanced** section. -1. Under **Change group URL**, enter a new name. -1. Select **Change group URL**. - -WARNING: -It is not possible to rename a namespace if it contains a -project with [Container Registry](../packages/container_registry/index.md) tags, -because the project cannot be moved. - -## Use a custom name for the initial branch - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43290) in GitLab 13.6. - -When you create a new project in GitLab, a default branch is created with the -first push. The group owner can -[customize the initial branch](../project/repository/branches/default.md#group-level-custom-initial-branch-name) -for the group's projects to meet your group's needs. - -## Remove a group - -To remove a group and its contents: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Settings > General**. -1. Expand the **Advanced** section. -1. In the **Remove group** section, select **Remove group**. -1. Type the group name. -1. Select **Confirm**. - -A group can also be removed from the groups dashboard: - -1. On the top bar, select **Menu > Groups**. -1. Select **Your Groups**. -1. Select (**{ellipsis_v}**) for the group you want to delete. -1. Select **Delete**. -1. In the Remove group section, select **Remove group**. -1. Type the group name. -1. Select **Confirm**. - -This action removes the group. It also adds a background job to delete all projects in the group. - -Specifically: - -- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). -- In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39504), if the user who sets up the deletion is removed from the group before the -deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. - -## Remove a group immediately **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336985) in GitLab 14.2. - -If you don't want to wait, you can remove a group immediately. - -Prerequisites: - -- You must have at least the Owner role for a group. -- You have [marked the group for deletion](#remove-a-group). - -To immediately remove a group marked for deletion: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Settings > General**. -1. Expand **Advanced**. -1. In the "Permanently remove group" section, select **Remove group**. -1. Confirm the action when asked to. - -Your group, its subgroups, projects, and all related resources, including issues and merge requests, -are deleted. - -## Restore a group **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33257) in GitLab 12.8. - -To restore a group that is marked for deletion: - -1. Go to your group's **Settings > General** page. -1. Expand the **Path, transfer, remove** section. -1. In the Restore group section, select **Restore group**. - -## Prevent group sharing outside the group hierarchy - -You can configure a top-level group so its subgroups and projects -cannot invite other groups outside of the top-level group's hierarchy. -This option is only available for top-level groups. - -For example, in the following group and project hierarchy: - -- **Animals > Dogs > Dog Project** -- **Animals > Cats** -- **Plants > Trees** - -If you prevent group sharing outside the hierarchy for the **Animals** group: - -- **Dogs** can invite the group **Cats**. -- **Dogs** cannot invite the group **Trees**. -- **Dog Project** can invite the group **Cats**. -- **Dog Project** cannot invite the group **Trees**. - -To prevent sharing outside of the group's hierarchy: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Settings > General**. -1. Expand **Permissions and group features**. -1. Select **Members cannot invite groups outside of `<group_name>` and its subgroups**. -1. Select **Save changes**. - -## Prevent a project from being shared with groups - -Prevent projects in a group from [sharing -a project with another group](../project/members/share_project_with_groups.md) to enable tighter control over project access. - -To prevent a project from being shared with other groups: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Select **Projects in `<group_name>` cannot be shared with other groups**. -1. Select **Save changes**. - -This setting applies to all subgroups unless overridden by a group owner. Groups already -added to a project lose access when the setting is enabled. - -## User cap for groups - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330027) in GitLab 14.7. - -FLAG: -On self-managed GitLab, this feature is not available. On GitLab.com, this feature is available for some groups. -This feature is not ready for production use. - -When the number of billable members reaches the user cap, new users can't be added to the group -without being approved by the group owner. - -Groups with the user cap feature enabled have [group sharing](#share-a-group-with-another-group) -disabled for the group and its subgroups. - -### Specify a user cap for a group - -Prerequisite: - -- You must be assigned the Owner role) for the group. - -To specify a user cap: - -1. On the top bar, select **Menu > Groups** and find your group. - You can set a cap on the top-level group only. -1. On the left sidebar, select **Settings > General**. -1. Expand **Permissions and group features**. -1. In the **User cap** box, enter the desired number of users. -1. Select **Save changes**. - -If you already have more users in the group than the user cap value, users -are not removed. However, you can't add more without approval. - -Increasing the user cap does not approve pending members. - -### Remove the user cap for a group - -You can remove the user cap, so there is no limit on the number of members you can add to a group. - -Prerequisite: - -- You must be assigned the Owner role) for the group. - -To remove the user cap: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Settings > General**. -1. Expand **Permissions and group features**. -1. In the **User cap** box, delete the value. -1. Select **Save changes**. - -Decreasing the user cap does not approve pending members. - -### Approve pending members for a group - -When the number of billable users reaches the user cap, any new member is put in a pending state -and must be approved. - -Pending members do not count as billable. Members count as billable only after they have been approved and are no longer in a pending state. - -Prerequisite: - -- You must be assigned the Owner role) for the group. - -To approve members that are pending because they've exceeded the user cap: - -1. On the top bar, select **Menu > Groups** and find your group. -1. On the left sidebar, select **Settings > Usage Quotas**. -1. On the **Seats** tab, under the alert, select **View pending approvals**. -1. For each member you want to approve, select **Approve**. - -## Prevent members from being added to projects in a group **(PREMIUM)** - -As a group owner, you can prevent any new project membership for all -projects in a group, allowing tighter control over project membership. - -For example, if you want to lock the group for an [Audit Event](../../administration/audit_events.md), -you can guarantee that project membership cannot be modified during the audit. - -You can still invite groups or to add members to groups, implicitly giving members access to projects in the **locked** group. - -The setting does not cascade. Projects in subgroups observe the subgroup configuration, ignoring the parent group. - -To prevent members from being added to projects in a group: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Under **Membership**, select **Users cannot be added to projects in this group**. -1. Select **Save changes**. - -All users who previously had permissions can no longer add members to a group. -API requests to add a new user to a project are not possible. - -## Export members as CSV **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/287940) in GitLab 14.2. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/336520) in GitLab 14.5. - -You can export a list of members in a group or subgroup as a CSV. - -1. Go to your group or subgroup and select either **Group information > Members** or **Subgroup information > Members**. -1. Select **Export as CSV**. -1. After the CSV file has been generated, it is emailed as an attachment to the user that requested it. - -## Group access restriction by IP address **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1985) in GitLab 12.0. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/215410) from GitLab Ultimate to GitLab Premium in 13.1. - -To ensure only people from your organization can access particular -resources, you can restrict access to groups by IP address. This group-level setting -applies to: - -- The GitLab UI, including subgroups, projects, issues, and pages. -- [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API. -- Using Git over SSH on GitLab.com. - -### Security implications - -You should consider some security implications before configuring IP address restrictions. - -- Administrators and group owners can access group settings from any IP address, regardless of IP restriction. However: - - Groups owners cannot access projects belonging to the group when accessing from a disallowed IP address. - - Administrators can access projects belonging to the group when accessing from a disallowed IP address. - Access to projects includes cloning code from them. - - Users can still see group and project names and hierarchies. Only the following are restricted: - - [Groups](../../api/groups.md), including all [group resources](../../api/api_resources.md#group-resources). - - [Project](../../api/projects.md), including all [project resources](../../api/api_resources.md#project-resources). -- When you register a runner, it is not bound by the IP restrictions. When the runner requests a new job or an update to - a job's state, it is also not bound by the IP restrictions. But when the running CI/CD job sends Git requests from a - restricted IP address, the IP restriction prevents code from being cloned. -- Users may still see some events from the IP restricted groups and projects on their dashboard. Activity may include - push, merge, issue, or comment events. -- IP access restrictions for Git operations via SSH are supported only on GitLab SaaS. - IP access restrictions applied to self-managed instances block SSH completely. - -### Restrict group access by IP address - -To restrict group access by IP address: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. In the **Restrict access by IP address** field, enter IPv4 or IPv6 address ranges in CIDR notation. -1. Select **Save changes**. - -In self-managed installations of GitLab 15.1 and later, you can also configure -[globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges) -at the group level. - -## Restrict group access by domain **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7297) in GitLab 12.2. -> - Support for specifying multiple email domains [added](https://gitlab.com/gitlab-org/gitlab/-/issues/33143) in GitLab 13.1. -> - Support for restricting access to projects in the group [added](https://gitlab.com/gitlab-org/gitlab/-/issues/14004) in GitLab 14.1.2. -> - Support for restricting group memberships to groups with a subset of the allowed email domains [added](https://gitlab.com/gitlab-org/gitlab/-/issues/354791) in GitLab 15.0.1 - -You can prevent users with email addresses in specific domains from being added to a group and its projects. - -To restrict group access by domain: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. In the **Restrict membership by email** field, enter the domain names. -1. Select **Save changes**. - -Any time you attempt to add a new user, the user's [primary email](../profile/index.md#change-your-primary-email) is compared against this list. -Only users with a [primary email](../profile/index.md#change-your-primary-email) that matches any of the configured email domain restrictions -can be added to the group. - -The most popular public email domains cannot be restricted, such as: - -- `gmail.com`, `yahoo.com`, `aol.com`, `icloud.com` -- `hotmail.com`, `hotmail.co.uk`, `hotmail.fr` -- `msn.com`, `live.com`, `outlook.com` - -When you share a group, both the source and target namespaces must allow the domains of the members' email addresses. - -## Restrict Git access protocols - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365601) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `group_level_git_protocol_control`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to -[enable the feature flag](../../administration/feature_flags.md) named `group_level_git_protocol_control`. On GitLab.com, -this feature is available. - -You can set the permitted protocols used to access a group's repositories to either SSH, HTTPS, or both. This setting -is disabled when the [instance setting](../admin_area/settings/visibility_and_access_controls.md#configure-enabled-git-access-protocols) is -configured by an administrator. - -To change the permitted Git access protocols for a group: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Choose the permitted protocols from **Enabled Git access protocols**. -1. Select **Save changes**. - -## Group file templates **(PREMIUM)** - -Use group file templates to share a set of templates for common file -types with every project in a group. It is analogous to the -[instance template repository](../admin_area/settings/instance_template_repository.md). -The selected project should follow the same naming conventions as -are documented on that page. - -You can only choose projects in the group as the template source. -This includes projects shared with the group, but it **excludes** projects in -subgroups or parent groups of the group being configured. - -You can configure this feature for both subgroups and immediate parent groups. A project -in a subgroup has access to the templates for that subgroup, as well as -any immediate parent groups. - -To learn how to create templates for issues and merge requests, see -[Description templates](../project/description_templates.md). - -Define project templates at a group level by setting a group as the template source. -[Learn more about group-level project templates](custom_project_templates.md). - -### Enable group file template **(PREMIUM)** - -To enable group file templates: - -1. Go to the group's **Settings > General** page. -1. Expand the **Templates** section. -1. Choose a project to act as the template repository. -1. Select **Save changes**. - -## Disable email notifications - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23585) in GitLab 12.2. - -You can disable all email notifications related to the group, which includes its subgroups and projects. - -To disable email notifications: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Select **Email notifications are disabled**. -1. Select **Save changes**. - -## Disable group mentions - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21301) in GitLab 12.6. - -You can prevent users from being added to a conversation and getting notified when -anyone [mentions a group](../discussions/index.md#mentions) -in which those users are members. - -Groups with disabled mentions are visualized accordingly in the autocompletion dropdown list. - -This is particularly helpful for groups with a large number of users. - -To disable group mentions: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Select **Group mentions are disabled**. -1. Select **Save changes**. - -## Enable delayed project deletion **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. -> - [Inheritance and enforcement added](https://gitlab.com/gitlab-org/gitlab/-/issues/321724) in GitLab 13.11. -> - [Instance setting to enable by default added](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2. -> - [Instance setting is inherited and enforced when disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. -> - [User interface changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352961) in GitLab 15.1. - -[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) is locked and disabled unless the instance-level settings for -[deletion protection](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) are enabled for either groups only or groups and projects. -When enabled on groups, projects in the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. -The default period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#retention-period). - -On self-managed GitLab, projects are deleted immediately by default. -In GitLab 14.2 and later, an administrator can -[change the default setting](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) -for projects in newly-created groups. - -On GitLab.com, see the [GitLab.com settings page](../gitlab_com/index.md#delayed-project-deletion) for -the default setting. - -To enable delayed deletion of projects in a group: - -1. Go to the group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Scroll to: - - (GitLab 15.1 and later) **Deletion protection** and select **Keep deleted projects**. - - (GitLab 15.0 and earlier) **Enable delayed project deletion** and tick the checkbox. -1. Optional. To prevent subgroups from changing this setting, select: - - (GitLab 15.1 and later), **Enforce deletion protection for all subgroups** - - (GitLab 15.0 and earlier), **Enforce for all subgroups**. -1. Select **Save changes**. - -NOTE: -In GitLab 13.11 and above the group setting for delayed project deletion is inherited by subgroups. As discussed in [Cascading settings](../../development/cascading_settings.md) inheritance can be overridden, unless enforced by an ancestor. - -## Prevent project forking outside group **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216987) in GitLab 13.3. - -By default, projects in a group can be forked. -Optionally, on [GitLab Premium](https://about.gitlab.com/pricing/) or higher tiers, -you can prevent the projects in a group from being forked outside of the current top-level group. - -This setting will be removed from the SAML setting page, and migrated to the -group settings page. In the interim period, both of these settings are taken into consideration. -If even one is set to `true`, then the group does not allow outside forks. - -To prevent projects from being forked outside the group: - -1. Go to the top-level group's **Settings > General** page. -1. Expand the **Permissions and group features** section. -1. Check **Prevent project forking outside current group**. -1. Select **Save changes**. - -Existing forks are not removed. - -## Group push rules **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34370) in GitLab 12.8. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/224129) in GitLab 13.4. - -Group push rules allow group maintainers to set -[push rules](../project/repository/push_rules.md) for newly created projects in the specific group. - -To configure push rules for a group: - -1. Go to the groups's **Push Rules** page. -1. Select the settings you want. -1. Select **Save Push Rules**. - -The group's new subgroups have push rules set for them based on either: - -- The closest parent group with push rules defined. -- Push rules set at the instance level, if no parent groups have push rules defined. - -## Group merge request approval settings **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285458) in GitLab 13.9. [Deployed behind the `group_merge_request_approval_settings_feature_flag` flag](../../administration/feature_flags.md), disabled by default. -> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) in GitLab 14.5. -> - [Feature flag `group_merge_request_approval_settings_feature_flag`](https://gitlab.com/gitlab-org/gitlab/-/issues/343872) removed in GitLab 14.9. - -Group approval settings manage [project merge request approval settings](../project/merge_requests/approvals/settings.md) -at the top-level group level. These settings [cascade to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) -that belong to the group. - -To view the merge request approval settings for a group: - -1. Go to the top-level group's **Settings > General** page. -1. Expand the **Merge request approvals** section. -1. Select the settings you want. -1. Select **Save changes**. - -Support for group-level settings for merge request approval rules is tracked in this [epic](https://gitlab.com/groups/gitlab-org/-/epics/4367). - ## Related topics - [Group wikis](../project/wiki/index.md) @@ -874,31 +56,8 @@ Support for group-level settings for merge request approval rules is tracked in - [Integrations](../admin_area/settings/project_integration_management.md). - [Transfer a project into a group](../project/settings/index.md#transfer-a-project-to-another-namespace). - [Share a project with a group](../project/members/share_project_with_groups.md): Give all group members access to the project at once. -- [Lock the sharing with group feature](#prevent-a-project-from-being-shared-with-groups). +- [Lock the sharing with group feature](access_and_permissions.md#prevent-a-project-from-being-shared-with-groups). - [Enforce two-factor authentication (2FA)](../../security/two_factor_authentication.md#enforce-2fa-for-all-users-in-a-group): Enforce 2FA for all group members. - Namespaces [API](../../api/namespaces.md) and [Rake tasks](../../raketasks/index.md). - [Control access and visibility](../admin_area/settings/visibility_and_access_controls.md). - -## Troubleshooting - -### Verify if access is blocked by IP restriction - -If a user sees a 404 when they would normally expect access, and the problem is limited to a specific group, search the `auth.log` rails log for one or more of the following: - -- `json.message`: `'Attempting to access IP restricted group'` -- `json.allowed`: `false` - -In viewing the log entries, compare the `remote.ip` with the list of -[allowed IPs](#group-access-restriction-by-ip-address) for the group. - -### Validation errors on namespaces and groups - -[GitLab 14.4 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70365) performs -the following checks when creating or updating namespaces or groups: - -- Namespaces must not have parents. -- Group parents must be groups and not namespaces. - -In the unlikely event that you see these errors in your GitLab installation, -[contact Support](https://about.gitlab.com/support/) so that we can improve this validation. diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 858b13b87b8..530635802a6 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -35,10 +35,22 @@ In GitLab, iterations are similar to milestones, with a few differences: ## View the iterations list -To view the iterations list, go to **{issues}** **Issues > Iterations**. +To view the iterations list: + +1. On the top bar, select **Menu > Projects** and find your project. +1. Select **Issues > Iterations**. + To view all the iterations in a cadence, ordered by descending date, select that iteration cadence. From there you can create a new iteration or select an iteration to get a more detailed view. +NOTE: +If a project has issue tracking +[turned off](../../project/settings/index.md#configure-project-visibility-features-and-permissions), +you can view the iterations list +by going to its URL. To do so, add: `/-/cadences` to your project or group URL. +For example `https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project/-/cadences`. +This is tracked in [issue 339009](https://gitlab.com/gitlab-org/gitlab/-/issues/339009). + ## Create an iteration > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/356069) in GitLab 14.10. @@ -175,6 +187,7 @@ To group issues by label: > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5077) in GitLab 14.1 [with a flag](../../../administration/feature_flags.md), named `iteration_cadences`. Disabled by default. > - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/354977) in GitLab 15.0: All scheduled iterations must start on the same day of the week as the cadence start day. Start date of cadence cannot be edited after the first iteration starts. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/354878) in GitLab 15.0. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/367493) in GitLab 15.3: A new automation start date can be selected for cadence. Upcoming iterations will be scheduled to start on the same day of the week as the changed start date. Iteration cadences automate iteration scheduling. You can use them to automate creating iterations every 1, 2, 3, or 4 weeks. You can also @@ -195,7 +208,7 @@ To create an iteration cadence: 1. Select **New iteration cadence**. 1. Complete the fields. - Enter the title and description of the iteration cadence. - - Enter the first iteration start date of the iteration cadence. Iterations will be scheduled to + - Select the automation start date of the iteration cadence. Iterations will be scheduled to begin on the same day of the week as the day of the week of the start date. - From the **Duration** dropdown list, select how many weeks each iteration should last. - From the **Upcoming iterations** dropdown list, select how many upcoming iterations should be @@ -215,10 +228,9 @@ To edit an iteration cadence: 1. On the left sidebar, select **Issues > Iterations**. 1. Select **Edit iteration cadence**. -When you edit the **Duration**, **Upcoming iterations**, or **First iteration start date** fields, -only upcoming iterations are affected. - -You can edit the first iteration start date of a cadence if the cadence has not started yet. +When you edit the **Automation start date** field, +you must set a new start date that doesn't overlap with the existing +current or past iterations. Editing **Upcoming iterations** is a non-destructive action. If ten upcoming iterations already exist, changing the number under **Upcoming iterations** to `2` @@ -261,25 +273,12 @@ To upgrade the iteration cadence to use the automation features: 1. On the top bar, select **Menu > Groups** and find your group. 1. On the left sidebar, select **Issues > Iterations**. 1. Select the three-dot menu (**{ellipsis_v}**) > **Edit cadence** for the cadence you want to upgrade. -1. Complete the required fields **Duration** and **Upcoming iterations**. +1. Complete the required fields **Duration**, **Upcoming iterations**, and **Automation start date**. +For **Automation start date**, you can select any date that doesn't overlap with the existing open iterations. +If you have upcoming iterations, the automatic scheduling adjusts them appropriately to fit +your chosen duration. 1. Select **Save changes**. -#### Start dates of converted cadences - -The first iteration start date of your converted cadence is set to the start date of its -**first** existing iteration. - -If you attempt to set a new start date, the conversion fails with an error message. -If your manual cadence is empty, converting it to use automatic scheduling is effectively -the same as creating a new automated cadence. - -GitLab will start scheduling new iterations on the same day of the week as the start date, -starting from the nearest such day from the current date. - -During the conversion process GitLab does not delete or modify existing **ongoing** or -**closed** iterations. If you have iterations with start dates in the future, -they are updated to fit your cadence settings. - #### Converted cadences example For example, suppose it's Friday, April 15, and you have three iterations in a manual cadence: @@ -288,20 +287,21 @@ For example, suppose it's Friday, April 15, and you have three iterations in a m - Tuesday, April 12 - Friday, April 15 (ongoing) - Tuesday, May 3 - Friday, May 6 (upcoming) -On Friday, April 15, you convert the manual cadence -to automate scheduling iterations every week, up to two upcoming iterations. - -The first iteration is closed, and the second iteration is ongoing, -so they aren't deleted or modified in the conversion process. - -To observe the weekly duration, the third iteration is updated so that it: +The earliest possible **Automation start date** you can choose +is Saturday, April 16 in this scenario, because April 15 overlaps with +the ongoing iteration. -- Starts on Monday, April 18 - which is the nearest date that is Monday. -- Ends on Sunday, April 24. - -Finally, to always have two upcoming iterations, an additional iteration is scheduled: +If you select Monday, April 18 as the automation start date to +automate scheduling iterations every week up to two upcoming iterations, +after the conversion you have the following iterations: - Monday, April 4 - Friday, April 8 (closed) - Tuesday, April 12 - Friday, April 15 (ongoing) - Monday, April 18 - Sunday, April 24 (upcoming) - Monday, April 25 - Sunday, May 1 (upcoming) + +Your existing upcoming iteration "Tuesday, April 12 - Friday, April 15" +is changed to "April 18 - Sunday, April 24". + +An additional upcoming iteration "April 25 - Sunday, May 1" is scheduled +to satisfy the requirement that there are at least two upcoming iterations scheduled. diff --git a/doc/user/group/manage.md b/doc/user/group/manage.md new file mode 100644 index 00000000000..a9cc6cc8432 --- /dev/null +++ b/doc/user/group/manage.md @@ -0,0 +1,570 @@ +--- +stage: Manage +group: Workspace +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Manage groups + +Use groups to manage one or more related projects at the same time. + +## View groups + +1. On the top bar, select **Menu > Groups**. +1. Select **Explore groups**. + +The **Groups** page shows a list of groups, sorted by last updated date. + +- To explore all public groups, select **Explore groups**. +- To view groups where you have a direct or indirect membership, select **Your groups**. This tab shows groups that you are a member of: + - Through membership of a subgroup's parent group. + - Through direct or inherited membership of a project in the group or subgroup. + +## Create a group + +To create a group: + +1. On the top bar, either: + - Select **Menu > Groups**, and on the right, select **Create group**. + - To the left of the search box, select the plus sign and then **New group**. +1. Select **Create group**. +1. Enter a name for the group in **Group name**. For a list of words that cannot be used as group names, see + [reserved names](../reserved_names.md). +1. Enter a path for the group in **Group URL**, which is used for the [namespace](../namespace/index.md). +1. Choose the [visibility level](../public_access.md). +1. Personalize your GitLab experience by answering the following questions: + - What is your role? + - Who will be using this group? + - What will you use this group for? +1. Invite GitLab members or other users to join the group. + +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +For details about groups, watch [GitLab Namespaces (users, groups and subgroups)](https://youtu.be/r0sJgjR2f5A). + +## Remove a group + +To remove a group and its contents: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Advanced** section. +1. In the **Remove group** section, select **Remove group**. +1. Type the group name. +1. Select **Confirm**. + +A group can also be removed from the groups dashboard: + +1. On the top bar, select **Menu > Groups**. +1. Select **Your Groups**. +1. Select (**{ellipsis_v}**) for the group you want to delete. +1. Select **Delete**. +1. In the Remove group section, select **Remove group**. +1. Type the group name. +1. Select **Confirm**. + +This action removes the group. It also adds a background job to delete all projects in the group. + +Specifically: + +- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). +- In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39504), if the user who sets up the deletion is removed from the group before the +deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. + +## Remove a group immediately **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336985) in GitLab 14.2. + +If you don't want to wait, you can remove a group immediately. + +Prerequisites: + +- You must have at least the Owner role for a group. +- You have [marked the group for deletion](#remove-a-group). + +To immediately remove a group marked for deletion: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. In the "Permanently remove group" section, select **Remove group**. +1. Confirm the action when asked to. + +Your group, its subgroups, projects, and all related resources, including issues and merge requests, +are deleted. + +## Restore a group **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33257) in GitLab 12.8. + +To restore a group that is marked for deletion: + +1. Go to your group's **Settings > General** page. +1. Expand the **Path, transfer, remove** section. +1. In the Restore group section, select **Restore group**. + +## Request access to a group + +As a user, you can request to be a member of a group, if an administrator allows it. + +1. On the top bar, select **Menu > Groups** and find your group. +1. Under the group name, select **Request Access**. + +As many as ten of the most-recently-active group owners receive an email with your request. +Any group owner can approve or decline the request. + +If you change your mind before your request is approved, select +**Withdraw Access Request**. + +## Filter and sort members in a group + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21727) in GitLab 12.6. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/228675) in GitLab 13.7. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289911) in GitLab 13.8. + +To find members in a group, you can sort, filter, or search. + +### Filter a group + +Filter a group to find members. By default, all members in the group and subgroups are displayed. + +1. Go to the group and select **Group information > Members**. +1. Above the list of members, in the **Filter members** box, enter filter criteria. + - To view members in the group only, select **Membership = Direct**. + - To view members of the group and its subgroups, select **Membership = Inherited**. + - To view members with two-factor authentication enabled or disabled, select **2FA = Enabled** or **Disabled**. + - [In GitLab 14.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/349887), to view GitLab users created by [SAML SSO](saml_sso/index.md) or [SCIM provisioning](saml_sso/scim_setup.md) select **Enterprise = true**. + +### Search a group + +You can search for members by name, username, or email. + +1. Go to the group and select **Group information > Members**. +1. Above the list of members, in the **Filter members** box, enter search criteria. +1. To the right of the **Filter members** box, select the magnifying glass (**{search}**). + +### Sort members in a group + +You can sort members by **Account**, **Access granted**, **Max role**, or **Last sign-in**. + +1. Go to the group and select **Group information > Members**. +1. Above the list of members, on the top right, from the **Account** list, select + the criteria to filter by. +1. To switch the sort between ascending and descending, to the right of the **Account** list, select the + arrow (**{sort-lowest}** or **{sort-highest}**). + +## Add users to a group + +You can give a user access to all projects in a group. + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Group information > Members**. +1. Select **Invite members**. +1. Fill in the fields. + - The role applies to all projects in the group. [Learn more about permissions](../permissions.md). + - On the **Access expiration date**, the user can no longer access projects in the group. +1. Select **Invite**. + +Members that are not automatically added are displayed on the **Invited** tab. +Users can be on this tab because they: + +- Have not yet accepted the invitation. +- Are waiting for [approval from an administrator](../admin_area/moderate_users.md). +- [Exceed the group user cap](#user-cap-for-groups). + +## Remove a member from the group + +Prerequisites: + +- You must have the Owner role. +- The member must have direct membership in the group. If + membership is inherited from a parent group, then the member can be removed + from the parent group only. + +To remove a member from a group: + +1. Go to the group. +1. From the left menu, select **Group information > Members**. +1. Next to the member you want to remove, select **Delete**. +1. Optional. On the **Remove member** confirmation box, select the + **Also unassign this user from linked issues and merge requests** checkbox. +1. Select **Remove member**. + +## Add projects to a group + +There are two different ways to add a new project to a group: + +- Select a group, and then select **New project**. You can then continue [creating your project](../../user/project/working_with_projects.md#create-a-project). +- While you are creating a project, select a group from the dropdown list. + +  + +### Specify who can add projects to a group + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2534) in GitLab 10.5. +> - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/25975) from GitLab Premium to GitLab Free in 11.10. + +By default, users with at least the Developer role can create projects under a group. + +To change this setting for a specific group: + +1. On the top bar, select **Menu > Groups**. +1. Select **Your Groups**. +1. Find the group and select it. +1. From the left menu, select **Settings > General**. +1. Expand the **Permissions and group features** section. +1. Select the desired option in the **Allowed to create projects** dropdown list. +1. Select **Save changes**. + +To change this setting globally, see [Default project creation protection](../admin_area/settings/visibility_and_access_controls.md#define-which-roles-can-create-projects). + +## Change the owner of a group + +You can change the owner of a group. Each group must always have at least one +member with the Owner role. + +- As an administrator: + 1. Go to the group and from the left menu, select **Group information > Members**. + 1. Give a different member the **Owner** role. + 1. Refresh the page. You can now remove the **Owner** role from the original owner. +- As the current group's owner: + 1. Go to the group and from the left menu, select **Group information > Members**. + 1. Give a different member the **Owner** role. + 1. Have the new owner sign in and remove the **Owner** role from you. + +## Change a group's path + +Changing a group's path (group URL) can have unintended side effects. Read +[how redirects behave](../project/repository/index.md#what-happens-when-a-repository-path-changes) +before you proceed. + +If you are changing the path so it can be claimed by another group or user, +you must rename the group too. Both names and paths must +be unique. + +To retain ownership of the original namespace and protect the URL redirects, +create a new group and transfer projects to it instead. + +To change your group path (group URL): + +1. Go to your group's **Settings > General** page. +1. Expand the **Advanced** section. +1. Under **Change group URL**, enter a new name. +1. Select **Change group URL**. + +WARNING: +It is not possible to rename a namespace if it contains a +project with [Container Registry](../packages/container_registry/index.md) tags, +because the project cannot be moved. + +## Change the default branch protection of a group + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7583) in GitLab 12.9. +> - [Settings moved and renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/340403) in GitLab 14.9. + +By default, every group inherits the branch protection set at the global level. + +To change this setting for a specific group, see [group level default branch protection](../project/repository/branches/default.md#group-level-default-branch-protection). + +To change this setting globally, see [initial default branch protection](../project/repository/branches/default.md#instance-level-default-branch-protection). + +NOTE: +In [GitLab Premium or higher](https://about.gitlab.com/pricing/), GitLab administrators can choose to [disable group owners from updating the default branch protection](../project/repository/branches/default.md#prevent-overrides-of-default-branch-protection). + +## Use a custom name for the initial branch + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43290) in GitLab 13.6. + +When you create a new project in GitLab, a default branch is created with the +first push. The group owner can +[customize the initial branch](../project/repository/branches/default.md#group-level-custom-initial-branch-name) +for the group's projects to meet your group's needs. + +## Share a group with another group + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18328) in GitLab 12.7. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 13.11 from a form to a modal window [with a flag](../feature_flags.md). Disabled by default. +> - Modal window [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/247208) in GitLab 14.8. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) in GitLab 14.9. + [Feature flag `invite_members_group_modal`](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) removed. + +Similar to how you [share a project with a group](../project/members/share_project_with_groups.md), +you can share a group with another group. To invite a group, you must be a member of it. Members get direct access +to the shared group. This includes members who inherited group membership from a parent group. + +To share a given group, for example, `Frontend` with another group, for example, +`Engineering`: + +1. Go to the `Frontend` group. +1. On the left sidebar, select **Group information > Members**. +1. Select **Invite a group**. +1. In the **Select a group to invite** list, select `Engineering`. +1. Select a [role](../permissions.md) as maximum access level. +1. Select **Invite**. + +After sharing the `Frontend` group with the `Engineering` group: + +- The **Groups** tab lists the `Engineering` group. +- The **Groups** tab lists a group regardless of whether it is a public or private group. +- All members of the `Engineering` group have access to the `Frontend` group. The same access levels of the members apply up to the maximum access level selected when sharing the group. + +## Transfer a group + +You can transfer groups in the following ways: + +- Transfer a subgroup to a new parent group. +- Convert a top-level group into a subgroup by transferring it to the desired group. +- Convert a subgroup into a top-level group by transferring it out of its current group. + +When transferring groups, note: + +- Changing a group's parent can have unintended side effects. See [what happens when a repository path changes](../project/repository/index.md#what-happens-when-a-repository-path-changes). +- You can only transfer groups to groups you manage. +- You must update your local repositories to point to the new location. +- If the immediate parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects change to match the new parent group's visibility. +- Only explicit group membership is transferred, not inherited membership. If the group's owners have only inherited membership, this leaves the group without an owner. In this case, the user transferring the group becomes the group's owner. +- Transfers fail if [packages](../packages/index.md) exist in any of the projects in the group, or in any of its subgroups. + +To transfer a group: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Advanced** section. +1. In the **Remove group** section, select **Transfer group**. +1. Select the group name in the drop down menu. +1. Select **Transfer group**. + +## Enable delayed project deletion **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) in GitLab 13.2. +> - [Inheritance and enforcement added](https://gitlab.com/gitlab-org/gitlab/-/issues/321724) in GitLab 13.11. +> - [Instance setting to enable by default added](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2. +> - [Instance setting is inherited and enforced when disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. +> - [User interface changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352961) in GitLab 15.1. + +[Delayed project deletion](../project/settings/index.md#delayed-project-deletion) is locked and disabled unless the instance-level settings for +[deletion protection](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) is enabled for either groups only or groups and projects. +When enabled on groups, projects in the group are deleted after a period of delay. During this period, projects are in a read-only state and can be restored. +The default period is seven days but [is configurable at the instance level](../admin_area/settings/visibility_and_access_controls.md#retention-period). + +On self-managed GitLab, projects are deleted immediately by default. +In GitLab 14.2 and later, an administrator can +[change the default setting](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) +for projects in newly-created groups. + +On GitLab.com, see the [GitLab.com settings page](../gitlab_com/index.md#delayed-project-deletion) for +the default setting. + +To enable delayed deletion of projects in a group: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Scroll to: + - (GitLab 15.1 and later) **Deletion protection** and select **Keep deleted projects**. + - (GitLab 15.0 and earlier) **Enable delayed project deletion** and tick the checkbox. +1. Optional. To prevent subgroups from changing this setting, select: + - (GitLab 15.1 and later), **Enforce deletion protection for all subgroups** + - (GitLab 15.0 and earlier), **Enforce for all subgroups**. +1. Select **Save changes**. + +NOTE: +In GitLab 13.11 and above the group setting for delayed project deletion is inherited by subgroups. As discussed in [Cascading settings](../../development/cascading_settings.md) inheritance can be overridden, unless enforced by an ancestor. + +## Disable email notifications + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23585) in GitLab 12.2. + +You can disable all email notifications related to the group, which includes its subgroups and projects. + +To disable email notifications: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Select **Disable email notifications**. +1. Select **Save changes**. + +## Disable group mentions + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21301) in GitLab 12.6. + +You can prevent users from being added to a conversation and getting notified when +anyone [mentions a group](../discussions/index.md#mentions) +in which those users are members. + +Groups with disabled mentions are visualized accordingly in the autocompletion dropdown list. + +This is particularly helpful for groups with a large number of users. + +To disable group mentions: + +1. Go to the group's **Settings > General** page. +1. Expand the **Permissions and group features** section. +1. Select **Disable group mentions**. +1. Select **Save changes**. + +## Export members as CSV **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/287940) in GitLab 14.2. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/336520) in GitLab 14.5. + +You can export a list of members in a group or subgroup as a CSV. + +1. Go to your group or subgroup and select either **Group information > Members** or **Subgroup information > Members**. +1. Select **Export as CSV**. +1. After the CSV file has been generated, it is emailed as an attachment to the user that requested it. + +## User cap for groups + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330027) in GitLab 14.7. + +FLAG: +On self-managed GitLab, this feature is not available. On GitLab.com, this feature is available for some groups. +This feature is not ready for production use. + +When the number of billable members reaches the user cap, new users can't be added to the group +without being approved by the group owner. + +Groups with the user cap feature enabled have [group sharing](#share-a-group-with-another-group) +disabled for the group and its subgroups. + +### Specify a user cap for a group + +Prerequisite: + +- You must be assigned the Owner role) for the group. + +To specify a user cap: + +1. On the top bar, select **Menu > Groups** and find your group. + You can set a cap on the top-level group only. +1. On the left sidebar, select **Settings > General**. +1. Expand **Permissions and group features**. +1. In the **User cap** box, enter the desired number of users. +1. Select **Save changes**. + +If you already have more users in the group than the user cap value, users +are not removed. However, you can't add more without approval. + +Increasing the user cap does not approve pending members. + +### Remove the user cap for a group + +You can remove the user cap, so there is no limit on the number of members you can add to a group. + +Prerequisite: + +- You must be assigned the Owner role) for the group. + +To remove the user cap: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Permissions and group features**. +1. In the **User cap** box, delete the value. +1. Select **Save changes**. + +Decreasing the user cap does not approve pending members. + +### Approve pending members for a group + +When the number of billable users reaches the user cap, any new member is put in a pending state +and must be approved. + +Pending members do not count as billable. Members count as billable only after they have been approved and are no longer in a pending state. + +Prerequisite: + +- You must be assigned the Owner role) for the group. + +To approve members that are pending because they've exceeded the user cap: + +1. On the top bar, select **Menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Usage Quotas**. +1. On the **Seats** tab, under the alert, select **View pending approvals**. +1. For each member you want to approve, select **Approve**. + +## Group file templates **(PREMIUM)** + +Use group file templates to share a set of templates for common file +types with every project in a group. It is analogous to the +[instance template repository](../admin_area/settings/instance_template_repository.md). +The selected project should follow the same naming conventions as +are documented on that page. + +You can only choose projects in the group as the template source. +This includes projects shared with the group, but it **excludes** projects in +subgroups or parent groups of the group being configured. + +You can configure this feature for both subgroups and immediate parent groups. A project +in a subgroup has access to the templates for that subgroup, as well as +any immediate parent groups. + +To learn how to create templates for issues and merge requests, see +[Description templates](../project/description_templates.md). + +Define project templates at a group level by setting a group as the template source. +[Learn more about group-level project templates](custom_project_templates.md). + +### Enable group file template **(PREMIUM)** + +To enable group file templates: + +1. Go to the group's **Settings > General** page. +1. Expand the **Templates** section. +1. Choose a project to act as the template repository. +1. Select **Save changes**. + +## Group merge request approval settings **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285458) in GitLab 13.9. [Deployed behind the `group_merge_request_approval_settings_feature_flag` flag](../../administration/feature_flags.md), disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) in GitLab 14.5. +> - [Feature flag `group_merge_request_approval_settings_feature_flag`](https://gitlab.com/gitlab-org/gitlab/-/issues/343872) removed in GitLab 14.9. + +Group approval settings manage [project merge request approval settings](../project/merge_requests/approvals/settings.md) +at the top-level group level. These settings [cascade to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) +that belong to the group. + +To view the merge request approval settings for a group: + +1. Go to the top-level group's **Settings > General** page. +1. Expand the **Merge request approvals** section. +1. Select the settings you want. +1. Select **Save changes**. + +Support for group-level settings for merge request approval rules is tracked in this [epic](https://gitlab.com/groups/gitlab-org/-/epics/4367). + +## Group activity analytics **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207164) in GitLab 12.10 as a [Beta feature](../../policy/alpha-beta-support.md#beta-features). + +For a group, you can view how many merge requests, issues, and members were created in the last 90 days. + +These Group Activity Analytics can be enabled with the `group_activity_analytics` [feature flag](../../development/feature_flags/index.md#enabling-a-feature-flag-locally-in-development). + + + +Changes to [group wikis](../project/wiki/group.md) do not appear in group activity analytics. + +### View group activity + +You can view the most recent actions taken in a group, either in your browser or in an RSS feed: + +1. On the top bar, select **Menu > Groups**. +1. Select **Your Groups**. +1. Find the group and select it. +1. On the left sidebar, select **Group information > Activity**. + +To view the activity feed in Atom format, select the +**RSS** (**{rss}**) icon. + +## Troubleshooting + +### Validation errors on namespaces and groups + +[GitLab 14.4 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70365) performs +the following checks when creating or updating namespaces or groups: + +- Namespaces must not have parents. +- Group parents must be groups and not namespaces. + +In the unlikely event that you see these errors in your GitLab installation, +[contact Support](https://about.gitlab.com/support/) so that we can improve this validation. diff --git a/doc/user/group/saml_sso/example_saml_config.md b/doc/user/group/saml_sso/example_saml_config.md new file mode 100644 index 00000000000..97e8f9c54a3 --- /dev/null +++ b/doc/user/group/saml_sso/example_saml_config.md @@ -0,0 +1,211 @@ +--- +stage: Manage +group: Authentication and Authorization +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +type: reference +--- + +# Example group SAML and SCIM configurations **(PREMIUM SAAS)** + +These are notes and screenshots regarding Group SAML and SCIM that the GitLab Support Team sometimes uses while troubleshooting, but which do not fit into the official documentation. GitLab is making this public, so that anyone can make use of the Support team's collected knowledge. + +Please refer to the GitLab [Group SAML](index.md) docs for information on the feature and how to set it up. + +When troubleshooting a SAML configuration, GitLab team members will frequently start with the [SAML troubleshooting section](index.md#troubleshooting). + +They may then set up a test configuration of the desired identity provider. We include example screenshots in this section. + +## SAML and SCIM screenshots + +This section includes relevant screenshots of the following example configurations of [Group SAML](index.md) and [Group SCIM](scim_setup.md): + +- [Azure Active Directory](#azure-active-directory) +- [Google Workspace](#google-workspace) +- [Okta](#okta) +- [OneLogin](#onelogin) + +WARNING: +These screenshots are updated only as needed by GitLab Support. They are **not** official documentation. + +If you are currently having an issue with GitLab, you may want to check your [support options](https://about.gitlab.com/support/). + +## Azure Active Directory + +Basic SAML app configuration: + + + +User claims and attributes: + + + +SCIM mapping: + + + + +Group Sync: + + + +## Google Workspace + +Basic SAML app configuration: + + + +User claims and attributes: + + + +IdP links and certificate: + +NOTE: +Google Workspace displays a SHA256 fingerprint. To retrieve the SHA1 fingerprint required by GitLab for configuring SAML, download the certificate and calculate the SHA1 certificate +fingerprint. + + + +## Okta + +Basic SAML app configuration for GitLab.com groups: + + + +Basic SAML app configuration for GitLab self-managed: + + + +User claims and attributes: + + + +Groups attribute: + + + +Advanced SAML app settings (defaults): + + + +IdP Links and Certificate: + + + +Sign on settings: + + + +Setting the username for the newly provisioned users when assigning them the SCIM app: + + + +## OneLogin + +Application details: + + + +Parameters: + + + +Adding a user: + + + +SSO settings: + + + +## SAML response example + +When a user signs in using SAML, GitLab receives a SAML response. The SAML response can be found in `production.log` logs as a base64-encoded message. Locate the response by +searching for `SAMLResponse`. The decoded SAML response is in XML format. For example: + +```xml +<?xml version="1.0" encoding="UTF-8"?> +<saml2p:Response xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol" xmlns:xs="http://www.w3.org/2001/XMLSchema" Destination="https://gitlabexample/-/saml/callback" ID="id4898983630840142426821432" InResponseTo="_c65e4c88-9425-4472-b42c-37f4186ac0ee" IssueInstant="2022-05-30T21:30:35.696Z" Version="2.0"> + <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" Format="urn:oasis:names:tc:SAML:2.0:nameid-format:entity">http://www.okta.com/exk2y6j57o1Pdr2lI8qh7</saml2:Issuer> + <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"> + <ds:SignedInfo> + <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/> + <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/> + <ds:Reference URI="#id4898983630840142426821432"> + <ds:Transforms> + <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/> + <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> + <ec:InclusiveNamespaces xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="xs"/> + </ds:Transform> + </ds:Transforms> + <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/> + <ds:DigestValue>neiQvv9d3OgS4GZW8Nptp4JhjpKs3GCefibn+vmRgk4=</ds:DigestValue> + </ds:Reference> + </ds:SignedInfo> + <ds:SignatureValue>dMsQX8ivi...HMuKGhyLRvabGU6CuPrf7==</ds:SignatureValue> + <ds:KeyInfo> + <ds:X509Data> + <ds:X509Certificate>MIIDq...cptGr3vN9TQ==</ds:X509Certificate> + </ds:X509Data> + </ds:KeyInfo> + </ds:Signature> + <saml2p:Status xmlns:saml2p="urn:oasis:names:tc:SAML:2.0:protocol"> + <saml2p:StatusCode Value="urn:oasis:names:tc:SAML:2.0:status:Success"/> + </saml2p:Status> + <saml2:Assertion xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" xmlns:xs="http://www.w3.org/2001/XMLSchema" ID="id489" IssueInstant="2022-05-30T21:30:35.696Z" Version="2.0"> + <saml2:Issuer xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" Format="urn:oasis:names:tc:SAML:2.0:nameid-format:entity">http://www.okta.com/exk2y6j57o1Pdr2lI8qh7</saml2:Issuer> + <ds:Signature xmlns:ds="http://www.w3.org/2000/09/xmldsig#"> + <ds:SignedInfo> + <ds:CanonicalizationMethod Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"/> + <ds:SignatureMethod Algorithm="http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"/> + <ds:Reference URI="#id48989836309833801859473359"> + <ds:Transforms> + <ds:Transform Algorithm="http://www.w3.org/2000/09/xmldsig#enveloped-signature"/> + <ds:Transform Algorithm="http://www.w3.org/2001/10/xml-exc-c14n#"> + <ec:InclusiveNamespaces xmlns:ec="http://www.w3.org/2001/10/xml-exc-c14n#" PrefixList="xs"/> + </ds:Transform> + </ds:Transforms> + <ds:DigestMethod Algorithm="http://www.w3.org/2001/04/xmlenc#sha256"/> + <ds:DigestValue>MaIsoi8hbT9gsi/mNZsz449mUuAcuEWY0q3bc4asOQs=</ds:DigestValue> + </ds:Reference> + </ds:SignedInfo> + <ds:SignatureValue>dMsQX8ivi...HMuKGhyLRvabGU6CuPrf7==<</ds:SignatureValue> + <ds:KeyInfo> + <ds:X509Data> + <ds:X509Certificate>MIIDq...cptGr3vN9TQ==</ds:X509Certificate> + </ds:X509Data> + </ds:KeyInfo> + </ds:Signature> + <saml2:Subject xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion"> + <saml2:NameID Format="urn:oasis:names:tc:SAML:2.0:nameid-format:persistent">useremail@domain.com</saml2:NameID> + <saml2:SubjectConfirmation Method="urn:oasis:names:tc:SAML:2.0:cm:bearer"> + <saml2:SubjectConfirmationData InResponseTo="_c65e4c88-9425-4472-b42c-37f4186ac0ee" NotOnOrAfter="2022-05-30T21:35:35.696Z" Recipient="https://gitlab.example.com/-/saml/callback"/> + </saml2:SubjectConfirmation> + </saml2:Subject> + <saml2:Conditions xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" NotBefore="2022-05-30T21:25:35.696Z" NotOnOrAfter="2022-05-30T21:35:35.696Z"> + <saml2:AudienceRestriction> + <saml2:Audience>https://gitlab.example.com/</saml2:Audience> + </saml2:AudienceRestriction> + </saml2:Conditions> + <saml2:AuthnStatement xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion" AuthnInstant="2022-05-30T21:30:35.696Z" SessionIndex="_c65e4c88-9425-4472-b42c-37f4186ac0ee"> + <saml2:AuthnContext> + <saml2:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml2:AuthnContextClassRef> + </saml2:AuthnContext> + </saml2:AuthnStatement> + <saml2:AttributeStatement xmlns:saml2="urn:oasis:names:tc:SAML:2.0:assertion"> + <saml2:Attribute Name="email" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> + <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">useremail@domain.com</saml2:AttributeValue> + </saml2:Attribute> + <saml2:Attribute Name="firtname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> + <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">John</saml2:AttributeValue> + </saml2:Attribute> + <saml2:Attribute Name="lastname" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> + <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">Doe</saml2:AttributeValue> + </saml2:Attribute> + <saml2:Attribute Name="Groups" NameFormat="urn:oasis:names:tc:SAML:2.0:attrname-format:unspecified"> + <saml2:AttributeValue xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="xs:string">Super-awesome-group</saml2:AttributeValue> + </saml2:Attribute> + </saml2:AttributeStatement> + </saml2:Assertion> +</saml2p:Response> +``` diff --git a/doc/user/group/saml_sso/group_sync.md b/doc/user/group/saml_sso/group_sync.md index b8b7a16b31b..8bc316f9396 100644 --- a/doc/user/group/saml_sso/group_sync.md +++ b/doc/user/group/saml_sso/group_sync.md @@ -28,7 +28,7 @@ To configure SAML Group Sync: 1. Ensure your SAML identity provider sends an attribute statement with the same name as the value of the `groups_attribute` setting. - For GitLab.com: 1. See [SAML SSO for GitLab.com groups](index.md). - 1. Ensure your SAML identity provider sends an attribute statement named `Groups` or `groups`. + 1. Ensure your SAML identity provider sends an attribute statement named `Groups` or `groups`. NOTE: The value for `Groups` or `groups` in the SAML response can be either the group name or the group ID. @@ -44,8 +44,8 @@ The value for `Groups` or `groups` in the SAML response can be either the group Other attribute names such as `http://schemas.microsoft.com/ws/2008/06/identity/claims/groups` are not accepted as a source of groups. -See the [SAML troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md) -for examples on configuring the required attribute name in the SAML identity provider's settings. +See [examples](../../../user/group/saml_sso/example_saml_config.md) +for configuring the required attribute name in the SAML identity provider's settings. ## Configure SAML Group Links @@ -161,3 +161,9 @@ graph TB GitLabGroupD --> |Member|GitLabUserC GitLabGroupD --> |Member|GitLabUserD ``` + +### Use the API + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/290367) in GitLab 15.3. + +You can use the GitLab API to [list, add, and delete](../../../api/groups.md#saml-group-links) SAML group links. diff --git a/doc/administration/troubleshooting/img/AzureAD-basic_SAML.png b/doc/user/group/saml_sso/img/AzureAD-basic_SAML.png Binary files differindex 7a0d83ab2dd..7a0d83ab2dd 100644 --- a/doc/administration/troubleshooting/img/AzureAD-basic_SAML.png +++ b/doc/user/group/saml_sso/img/AzureAD-basic_SAML.png diff --git a/doc/administration/troubleshooting/img/AzureAD-claims.png b/doc/user/group/saml_sso/img/AzureAD-claims.png Binary files differindex 576040be337..576040be337 100644 --- a/doc/administration/troubleshooting/img/AzureAD-claims.png +++ b/doc/user/group/saml_sso/img/AzureAD-claims.png diff --git a/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png b/doc/user/group/saml_sso/img/AzureAD-scim_attribute_mapping.png Binary files differindex 5ad82003eed..5ad82003eed 100644 --- a/doc/administration/troubleshooting/img/AzureAD-scim_attribute_mapping.png +++ b/doc/user/group/saml_sso/img/AzureAD-scim_attribute_mapping.png diff --git a/doc/administration/troubleshooting/img/AzureAD-scim_provisioning.png b/doc/user/group/saml_sso/img/AzureAD-scim_provisioning.png Binary files differindex b2c385151a6..b2c385151a6 100644 --- a/doc/administration/troubleshooting/img/AzureAD-scim_provisioning.png +++ b/doc/user/group/saml_sso/img/AzureAD-scim_provisioning.png diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png b/doc/user/group/saml_sso/img/GoogleWorkspace-basic-SAML_v14_10.png Binary files differindex e002503ddc0..e002503ddc0 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-basic-SAML_v14_10.png +++ b/doc/user/group/saml_sso/img/GoogleWorkspace-basic-SAML_v14_10.png diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png b/doc/user/group/saml_sso/img/GoogleWorkspace-claims_v14_10.png Binary files differindex 2b827e122a8..2b827e122a8 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-claims_v14_10.png +++ b/doc/user/group/saml_sso/img/GoogleWorkspace-claims_v14_10.png diff --git a/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png b/doc/user/group/saml_sso/img/GoogleWorkspace-linkscert_v14_10.png Binary files differindex 8d6c5010670..8d6c5010670 100644 --- a/doc/administration/troubleshooting/img/GoogleWorkspace-linkscert_v14_10.png +++ b/doc/user/group/saml_sso/img/GoogleWorkspace-linkscert_v14_10.png diff --git a/doc/user/group/saml_sso/img/Okta-GroupAttribute.png b/doc/user/group/saml_sso/img/Okta-GroupAttribute.png Binary files differnew file mode 100644 index 00000000000..54c69053754 --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-GroupAttribute.png diff --git a/doc/user/group/saml_sso/img/Okta-GroupSAML.png b/doc/user/group/saml_sso/img/Okta-GroupSAML.png Binary files differnew file mode 100644 index 00000000000..7871e4ff82b --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-GroupSAML.png diff --git a/doc/user/group/saml_sso/img/Okta-SM.png b/doc/user/group/saml_sso/img/Okta-SM.png Binary files differnew file mode 100644 index 00000000000..b335009fed9 --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-SM.png diff --git a/doc/user/group/saml_sso/img/Okta-advancedsettings.png b/doc/user/group/saml_sso/img/Okta-advancedsettings.png Binary files differnew file mode 100644 index 00000000000..1478dc58ccd --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-advancedsettings.png diff --git a/doc/user/group/saml_sso/img/Okta-attributes.png b/doc/user/group/saml_sso/img/Okta-attributes.png Binary files differnew file mode 100644 index 00000000000..38af72474d8 --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-attributes.png diff --git a/doc/user/group/saml_sso/img/Okta-linkscert.png b/doc/user/group/saml_sso/img/Okta-linkscert.png Binary files differnew file mode 100644 index 00000000000..62db5d2b7e3 --- /dev/null +++ b/doc/user/group/saml_sso/img/Okta-linkscert.png diff --git a/doc/administration/troubleshooting/img/OneLogin-SSOsettings.png b/doc/user/group/saml_sso/img/OneLogin-SSOsettings.png Binary files differindex 58f936d8567..58f936d8567 100644 --- a/doc/administration/troubleshooting/img/OneLogin-SSOsettings.png +++ b/doc/user/group/saml_sso/img/OneLogin-SSOsettings.png diff --git a/doc/administration/troubleshooting/img/OneLogin-app_details.png b/doc/user/group/saml_sso/img/OneLogin-app_details.png Binary files differindex 77618960897..77618960897 100644 --- a/doc/administration/troubleshooting/img/OneLogin-app_details.png +++ b/doc/user/group/saml_sso/img/OneLogin-app_details.png diff --git a/doc/administration/troubleshooting/img/OneLogin-parameters.png b/doc/user/group/saml_sso/img/OneLogin-parameters.png Binary files differindex a2fa734152c..a2fa734152c 100644 --- a/doc/administration/troubleshooting/img/OneLogin-parameters.png +++ b/doc/user/group/saml_sso/img/OneLogin-parameters.png diff --git a/doc/administration/troubleshooting/img/OneLogin-userAdd.png b/doc/user/group/saml_sso/img/OneLogin-userAdd.png Binary files differindex 54c1ecd2e68..54c1ecd2e68 100644 --- a/doc/administration/troubleshooting/img/OneLogin-userAdd.png +++ b/doc/user/group/saml_sso/img/OneLogin-userAdd.png diff --git a/doc/administration/troubleshooting/img/azure_configure_group_claim.png b/doc/user/group/saml_sso/img/azure_configure_group_claim.png Binary files differindex 9d8c5348273..9d8c5348273 100644 --- a/doc/administration/troubleshooting/img/azure_configure_group_claim.png +++ b/doc/user/group/saml_sso/img/azure_configure_group_claim.png diff --git a/doc/user/group/saml_sso/img/okta_saml_settings.png b/doc/user/group/saml_sso/img/okta_saml_settings.png Binary files differnew file mode 100644 index 00000000000..9c774b72d66 --- /dev/null +++ b/doc/user/group/saml_sso/img/okta_saml_settings.png diff --git a/doc/administration/troubleshooting/img/okta_setting_username.png b/doc/user/group/saml_sso/img/okta_setting_username.png Binary files differindex 5f87d14bb8b..5f87d14bb8b 100644 --- a/doc/administration/troubleshooting/img/okta_setting_username.png +++ b/doc/user/group/saml_sso/img/okta_setting_username.png diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index 80e7a5903fa..25060f8e749 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -99,7 +99,7 @@ After you set up your identity provider to work with GitLab, you must configure  NOTE: -The certificate [fingerprint algorithm](../../../integration/saml.md#notes-on-configuring-your-identity-provider) must be in SHA1. When configuring the identity provider, use a secure signature algorithm. +The certificate [fingerprint algorithm](../../../integration/saml.md#notes-on-configuring-your-identity-provider) must be in SHA1. When configuring the identity provider (such as [Google Workspace](#google-workspace-setup-notes)), use a secure signature algorithm. ### SSO enforcement @@ -131,6 +131,8 @@ SSO has the following effects when enabled: - Git activity originating from CI/CD jobs do not have the SSO check enforced. - Credentials that are not tied to regular users (for example, project and group access tokens, and deploy keys) do not have the SSO check enforced. - Users must be signed-in through SSO before they can pull images using the [Dependency Proxy](../../packages/dependency_proxy/index.md). +- When the **Enforce SSO-only authentication for Git and Dependency Proxy activity for this group** option is enabled, any API endpoint that involves Git activity is under SSO + enforcement. For example, creating or deleting a branch, commit, or tag. When SSO is enforced, users are not immediately revoked. If the user: @@ -174,7 +176,7 @@ The recommended attributes and claims settings are: If using [Group Sync](#group-sync), customize the name of the group claim to match the required attribute. -See the [troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory) for an example configuration. +See our [example configuration page](example_saml_config.md#azure-active-directory). ### Google Workspace setup notes @@ -191,7 +193,7 @@ with the notes below for consideration. NOTE: Google Workspace displays a SHA256 fingerprint. To retrieve the SHA1 fingerprint required by GitLab for [configuring SAML](#configure-gitlab), download the certificate and calculate -the SHA1 certificate fingerprint. +the SHA1 certificate fingerprint using this sample command: `openssl x509 -noout -fingerprint -sha1 -inform pem -in "GoogleIDPCertificate-domain.com.pem"`. The recommended attributes and claims settings are: @@ -206,7 +208,7 @@ For NameID, the following settings are recommended: When selecting **Verify SAML Configuration** on the GitLab SAML SSO page, disregard the warning recommending setting the NameID format to "persistent". -See the [troubleshooting page](../../../administration/troubleshooting/group_saml_scim.md#google-workspace) for an example configuration. +See our [example configuration page](example_saml_config.md#google-workspace). ### Okta setup notes @@ -445,7 +447,7 @@ To generate a SAML Response: ### Verifying configuration -For convenience, we've included some [example resources](../../../administration/troubleshooting/group_saml_scim.md) used by our Support Team. While they may help you verify the SAML app configuration, they are not guaranteed to reflect the current state of third-party products. +For convenience, we've included some [example resources](../../../user/group/saml_sso/example_saml_config.md) used by our Support Team. While they may help you verify the SAML app configuration, they are not guaranteed to reflect the current state of third-party products. ### Verifying NameID diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index 04aa99e08af..7962f171166 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -82,7 +82,7 @@ For a list of required attributes, refer to the [SCIM API documentation](../../. | `userPrincipalName` | `emails[type eq "work"].value` | | | `mailNickname` | `userName` | | -For guidance, you can view [an example configuration in the troubleshooting reference](../../../administration/troubleshooting/group_saml_scim.md#azure-active-directory). +For guidance, you can view [an example configuration](example_saml_config.md#azure-active-directory). 1. Below the mapping list select **Show advanced options > Edit attribute list for AppName**. 1. Ensure the `id` is the primary and required field, and `externalId` is also required. @@ -106,7 +106,7 @@ Before you start this section: - Check that you are using Okta [Lifecycle Management](https://www.okta.com/products/lifecycle-management/) product. This product tier is required to use SCIM on Okta. To check which Okta product you are using, check your signed Okta contract, contact your Okta AE, CSM, or Okta support. - Complete the [GitLab configuration](#gitlab-configuration) process. -- Complete the setup for SAML application for [Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/overview/), as described in the [Okta setup notes](index.md#okta-setup-notes). +- Complete the setup for SAML application for [Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/main/), as described in the [Okta setup notes](index.md#okta-setup-notes). - Check that your Okta SAML setup matches our documentation exactly, especially the NameID configuration. Otherwise, the Okta SCIM app may not work properly. After the above steps are complete: @@ -220,6 +220,8 @@ It is important that this SCIM `id` and SCIM `externalId` are configured to the ### How do I verify user's SAML NameId matches the SCIM externalId +Admins can use the Admin Area to [list SCIM identities for a user](../../admin_area/#user-identities). + Group owners can see the list of users and the `externalId` stored for each user in the group SAML SSO Settings page. A possible alternative is to use the [SCIM API](../../../api/scim.md#get-a-list-of-scim-provisioned-users) to manually retrieve the `externalId` we have stored for users, also called the `external_uid` or `NameId`. diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index 9e8fc120731..c3098bb56c2 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -154,5 +154,8 @@ to groups instead of projects. Bot users for groups: - Do not count as licensed seats. - Can have a maximum role of Owner for a group. For more information, see [Create a group access token](../../../api/group_access_tokens.md#create-a-group-access-token). +- The username is set to `group_{project_id}_bot` for the first access token. For example, `project_123_bot`. +- The email is set to `group{group_id}_bot@noreply.{Gitlab.config.gitlab.host}`. For example, `group123_bot@noreply.example.com`. +- All other properties are similar to [bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects) For more information, see [Bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects). diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index bf4e13779fd..12434de5efc 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -47,6 +47,28 @@ graph TD end ``` +## View subgroups of a group + +Prerequisite: + +- To view private nested subgroups, you must be a direct or inherited member of +the private subgroup. + +To view the subgroups of a group: + +1. On the top bar, select **Menu > Groups** and find your group. +1. Select the **Subgroups and projects** tab. +1. To view a nested subgroup, expand a subgroup in the hierarchy list. + +### Private subgroups in public parent groups + +In the hierarchy list, public groups with a private subgroup have an expand option (**{chevron-down}**) +for all users that indicate there is a subgroup. When users who are not direct or inherited members of +the private subgroup select expand (**{chevron-down}**), the nested subgroup does not display. + +If you prefer to keep information about the presence of nested subgroups private, we advise that you only +add private subgroups to private parent groups. + ## Create a subgroup Prerequisites: @@ -102,7 +124,7 @@ Subgroup members can: 1. Be [direct members](../../project/members/index.md#add-users-to-a-project) of the subgroup. 1. [Inherit membership](../../project/members/index.md#inherited-membership) of the subgroup from the subgroup's parent group. -1. Be a member of a group that was [shared with the subgroup's top-level group](../index.md#share-a-group-with-another-group). +1. Be a member of a group that was [shared with the subgroup's top-level group](../manage.md#share-a-group-with-another-group). ```mermaid flowchart RL @@ -161,7 +183,7 @@ In the screenshot above: - Administrator has the Owner role on group _Four_ and is a member of all subgroups. For that reason, as with User 3, the **Source** column indicates they are a direct member. -Members can be [filtered by inherited or direct membership](../index.md#filter-a-group). +Members can be [filtered by inherited or direct membership](../manage.md#filter-a-group). ### Override ancestor group membership diff --git a/doc/user/img/completed_tasks_v13_3.png b/doc/user/img/completed_tasks_v13_3.png Binary files differdeleted file mode 100644 index b12d95f0a23..00000000000 --- a/doc/user/img/completed_tasks_v13_3.png +++ /dev/null diff --git a/doc/user/img/completed_tasks_v15_3.png b/doc/user/img/completed_tasks_v15_3.png Binary files differnew file mode 100644 index 00000000000..c1619e70839 --- /dev/null +++ b/doc/user/img/completed_tasks_v15_3.png diff --git a/doc/user/infrastructure/clusters/connect/new_civo_cluster.md b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md index fad75ca6cab..ecf93958b1e 100644 --- a/doc/user/infrastructure/clusters/connect/new_civo_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md @@ -90,7 +90,7 @@ Refer to the [Civo Terraform provider](https://registry.terraform.io/providers/c After configuring your project, manually trigger the provisioning of your cluster. In GitLab: 1. On the left sidebar, go to **CI/CD > Pipelines**. -1. Next to **Play** (**{play}**), select the dropdown icon (**{angle-down}**). +1. Next to **Play** (**{play}**), select the dropdown icon (**{chevron-lg-down}**). 1. Select **Deploy** to manually trigger the deployment job. When the pipeline finishes successfully, you can see your new cluster: diff --git a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md index 798e02ab9b4..2f5967bd7ee 100644 --- a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md @@ -122,12 +122,13 @@ To remove all resources: stages: - init - validate + - test - build - deploy - cleanup destroy: - extends: .destroy + extends: .terraform:destroy needs: [] ``` diff --git a/doc/user/infrastructure/clusters/deploy/inventory_object.md b/doc/user/infrastructure/clusters/deploy/inventory_object.md index d184d969ace..fc3b86776f0 100644 --- a/doc/user/infrastructure/clusters/deploy/inventory_object.md +++ b/doc/user/infrastructure/clusters/deploy/inventory_object.md @@ -4,9 +4,10 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Tracking cluster resources managed by GitLab **(PREMIUM)** +# Tracking cluster resources managed by GitLab **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/346567) from GitLab Premium to GitLab Free in 15.3. GitLab uses an inventory object to track the resources you deploy to your cluster. The inventory object is a `ConfigMap` that contains a list of controlled objects. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md index 5ad1fb81a39..51fd626ce0f 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md @@ -18,19 +18,28 @@ uncomment this line from your `helmfile.yaml`: - path: applications/cert-manager/helmfile.yaml ``` +And update the `applications/cert-manager/helmfile.yaml` with a valid email address. + +```yaml + values: + - letsEncryptClusterIssuer: + # + # IMPORTANT: This value MUST be set to a valid email. + # + email: example@example.com +``` + NOTE: -If your Kubernetes version is earlier than 1.20 and you are [migrating from GitLab -Managed Apps to a cluster management -project](../../../../clusters/migrating_from_gma_to_project_template.md), then -you can instead use `- path: applications/cert-manager-legacy/helmfile.yaml` to +If your Kubernetes version is earlier than 1.20 and you are +[migrating from GitLab Managed Apps to a cluster management project](../../../../clusters/migrating_from_gma_to_project_template.md), +then you can instead use `- path: applications/cert-manager-legacy/helmfile.yaml` to take over an existing release of cert-manager v0.10. cert-manager: - Is installed by default into the `gitlab-managed-apps` namespace of your cluster. - Includes a - [Let's Encrypt - `ClusterIssuer`](https://cert-manager.io/docs/configuration/acme/) enabled by + [Let's Encrypt `ClusterIssuer`](https://cert-manager.io/docs/configuration/acme/) enabled by default. In the `certmanager-issuer` release, the issuer requires a valid email address for `letsEncryptClusterIssuer.email`. Let's Encrypt uses this email address to contact you about expiring certificates and issues related to your account. diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md b/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md index 383e857bb20..64d325dedc6 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/prometheus.md @@ -1,27 +1,11 @@ --- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../../../../operations/metrics/index.md' +remove_date: '2022-11-03' --- -# Install Prometheus with a cluster management project **(FREE)** +This document was moved to [another location](../../../../../operations/metrics/index.md). -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -[Prometheus](https://prometheus.io/docs/introduction/overview/) is an -open-source monitoring and alerting system for supervising your -deployed applications. - -Assuming you already have a project created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install Prometheus you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/prometheus/helmfile.yaml -``` - -You can customize the installation of Prometheus by updating the -`applications/prometheus/values.yaml` file in your cluster -management project. Refer to the -[Configuration section](https://github.com/helm/charts/tree/master/stable/prometheus#configuration) -of the Prometheus chart's README for the available configuration options. +<!-- This redirect file can be deleted after <2022-11-03>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md b/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md index d2d314b649e..f42e9c83120 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/sentry.md @@ -1,70 +1,11 @@ --- -stage: Monitor -group: Respond -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../../../../../operations/error_tracking.md' +remove_date: '2022-11-03' --- -# Install Sentry with a cluster management project **(FREE)** +This document was moved to [another location](../../../../../operations/error_tracking.md). -> [Introduced](https://gitlab.com/gitlab-org/project-templates/cluster-management/-/merge_requests/5) in GitLab 14.0. - -The Sentry Helm chart [recommends](https://github.com/helm/charts/blob/f6e5784f265dd459c5a77430185d0302ed372665/stable/sentry/values.yaml#L284-L285) -at least 3 GB of available RAM for database migrations. - -Assuming you already have a project created from a -[management project template](../../../../../user/clusters/management_project_template.md), to install Sentry you should -uncomment this line from your `helmfile.yaml`: - -```yaml - - path: applications/sentry/helmfile.yaml -``` - -Sentry is installed by default into the `gitlab-managed-apps` namespace -of your cluster. - -You can customize the installation of Sentry by defining -`applications/sentry/values.yaml` file in your cluster -management project. Refer to the -[chart](https://github.com/helm/charts/tree/master/stable/sentry) -for the available configuration options. - -We recommend you pay close attention to the following configuration options: - -- `email`. Needed to invite users to your Sentry instance and to send error emails. -- `user`. Where you can set the login credentials for the default administrator user. -- `postgresql`. For a PostgreSQL password that can be used when running future updates. - -When upgrading, it's important to provide the existing PostgreSQL password (given -using the `postgresql.postgresqlPassword` key) to avoid authentication errors. -Read the [PostgreSQL chart documentation](https://github.com/helm/charts/tree/master/stable/postgresql#upgrade) -for more information. - -Here is an example configuration for Sentry: - -```yaml -# Admin user to create -user: - # Indicated to create the admin user or not, - # Default is true as the initial installation. - create: true - email: "<your email>" - password: "<your password>" - -email: - from_address: "<your from email>" - host: smtp - port: 25 - use_tls: false - user: "<your email username>" - password: "<your email password>" - enable_replies: false - -ingress: - enabled: true - hostname: "<sentry.example.com>" - -# Needs to be here between runs. -# See https://github.com/helm/charts/tree/master/stable/postgresql#upgrade for more info -postgresql: - postgresqlPassword: example-postgresql-password -``` +<!-- This redirect file can be deleted after <2022-11-03>. --> +<!-- Redirects that point to other docs in the same project expire in three months. --> +<!-- Redirects that point to docs in a different project or site (link is not relative and starts with `https:`) expire in one year. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/redirects.html --> diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md index 06e67b78c91..31a22a240d9 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md @@ -35,7 +35,7 @@ Vault application causes downtime. To optimally use Vault in a production environment, it's ideal to have a good understanding of the internals of Vault and how to configure it. This can be done by reading -the [Vault Configuration guide](../../../../../ci/secrets/#configure-your-vault-server), +the [Vault Configuration guide](../../../../../ci/secrets/index.md#configure-your-vault-server), the [Vault documentation](https://www.vaultproject.io/docs/internals) and the Vault Helm chart [`values.yaml` file](https://github.com/hashicorp/vault-helm/blob/v0.3.3/values.yaml). diff --git a/doc/user/infrastructure/iac/terraform_state.md b/doc/user/infrastructure/iac/terraform_state.md index 24203e8d922..4e78e0bbed5 100644 --- a/doc/user/infrastructure/iac/terraform_state.md +++ b/doc/user/infrastructure/iac/terraform_state.md @@ -68,17 +68,34 @@ To configure GitLab CI/CD as a backend: } ``` -1. In the root directory of your project repository, create a `.gitlab-ci.yml` file. Use - [this file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) - to populate it. - +1. In the root directory of your project repository, create a `.gitlab-ci.yml` file. Use the + [`Terraform.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) + template to populate it. 1. Push your project to GitLab. This action triggers a pipeline, which runs the `gitlab-terraform init`, `gitlab-terraform validate`, and `gitlab-terraform plan` commands. -1. Trigger the manual `terraform apply` job from the previous pipeline to provision the defined infrastructure. +1. Trigger the manual `deploy` job from the previous pipeline, which runs `gitlab-terraform apply` command, to provision the defined infrastructure. The output from the above `terraform` commands should be viewable in the job logs. +The `gitlab-terraform` CLI is a wrapper around the `terraform` CLI. You can [view the source code of `gitlab-terraform`](https://gitlab.com/gitlab-org/terraform-images/-/blob/master/src/bin/gitlab-terraform.sh) if you're interested. + +If you prefer to call the `terraform` commands explicitly, you can override +the template, and instead, use it as reference for what you can achieve. + +### Customizing your Terraform environment variables + +When you use the `Terraform.gitlab-ci.yml` template, you can use [Terraform HTTP configuration variables](https://www.terraform.io/language/settings/backends/http#configuration-variables) when you define your CI/CD jobs. + +To customize your `terraform init` and override the Terraform configuration, +use environment variables instead of the `terraform init -backend-config=...` approach. +When you use `-backend-config`, the configuration is: + +- Cached in the output of the `terraform plan` command. +- Usually passed forward to the `terraform apply` command. + +This configuration can lead to problems like [being unable to lock Terraform state files in CI jobs](troubleshooting.md#unable-to-lock-terraform-state-files-in-ci-jobs-for-terraform-apply-using-a-plan-created-in-a-previous-job). + ## Access the state from your local machine You can access the GitLab-managed Terraform state from your local machine. @@ -109,66 +126,6 @@ You should use a local terminal to run the commands needed for migrating to GitL The following example demonstrates how to change the state name. The same workflow is needed to migrate to GitLab-managed Terraform state from a different state storage backend. -## Use your GitLab backend as a remote data source - -You can use a GitLab-managed Terraform state backend as a -[Terraform data source](https://www.terraform.io/language/state/remote-state-data). - -1. In your `main.tf` or other relevant file, declare these variables. Leave the values empty. - - ```hcl - variable "example_remote_state_address" { - type = string - description = "Gitlab remote state file address" - } - - variable "example_username" { - type = string - description = "Gitlab username to query remote state" - } - - variable "example_access_token" { - type = string - description = "GitLab access token to query remote state" - } - ``` - -1. To override the values from the previous step, create a file named `example.auto.tfvars`. This file should **not** be versioned in your project repository. - - ```plaintext - example_remote_state_address = "https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>" - example_username = "<GitLab username>" - example_access_token = "<GitLab Personal Access Token>" - ``` - -1. In a `.tf` file, define the data source by using [Terraform input variables](https://www.terraform.io/language/values/variables): - - ```hcl - data "terraform_remote_state" "example" { - backend = "http" - - config = { - address = var.example_remote_state_address - username = var.example_username - password = var.example_access_token - } - } - ``` - - - **address**: The URL of the remote state backend you want to use as a data source. - For example, `https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>`. - - **username**: The username to authenticate with the data source. If you are using - a [Personal Access Token](../../profile/personal_access_tokens.md) for - authentication, this value is your GitLab username. If you are using GitLab CI/CD, this value is `'gitlab-ci-token'`. - - **password**: The password to authenticate with the data source. If you are using a Personal Access Token for - authentication, this value is the token value (the token must have the **API** scope). - If you are using GitLab CI/CD, this value is the contents of the `${CI_JOB_TOKEN}` CI/CD variable. - -Outputs from the data source can now be referenced in your Terraform resources -using `data.terraform_remote_state.example.outputs.<OUTPUT-NAME>`. - -To read the Terraform state in the target project, you need at least the Developer role. - ### Set up the initial backend ```shell @@ -264,6 +221,66 @@ commands will detect it and remind you to do so if necessary. If you type `yes`, it copies your state from the old location to the new location. You can then go back to running it in GitLab CI/CD. +## Use your GitLab backend as a remote data source + +You can use a GitLab-managed Terraform state backend as a +[Terraform data source](https://www.terraform.io/language/state/remote-state-data). + +1. In your `main.tf` or other relevant file, declare these variables. Leave the values empty. + + ```hcl + variable "example_remote_state_address" { + type = string + description = "Gitlab remote state file address" + } + + variable "example_username" { + type = string + description = "Gitlab username to query remote state" + } + + variable "example_access_token" { + type = string + description = "GitLab access token to query remote state" + } + ``` + +1. To override the values from the previous step, create a file named `example.auto.tfvars`. This file should **not** be versioned in your project repository. + + ```plaintext + example_remote_state_address = "https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>" + example_username = "<GitLab username>" + example_access_token = "<GitLab Personal Access Token>" + ``` + +1. In a `.tf` file, define the data source by using [Terraform input variables](https://www.terraform.io/language/values/variables): + + ```hcl + data "terraform_remote_state" "example" { + backend = "http" + + config = { + address = var.example_remote_state_address + username = var.example_username + password = var.example_access_token + } + } + ``` + + - **address**: The URL of the remote state backend you want to use as a data source. + For example, `https://gitlab.com/api/v4/projects/<TARGET-PROJECT-ID>/terraform/state/<TARGET-STATE-NAME>`. + - **username**: The username to authenticate with the data source. If you are using + a [Personal Access Token](../../profile/personal_access_tokens.md) for + authentication, this value is your GitLab username. If you are using GitLab CI/CD, this value is `'gitlab-ci-token'`. + - **password**: The password to authenticate with the data source. If you are using a Personal Access Token for + authentication, this value is the token value (the token must have the **API** scope). + If you are using GitLab CI/CD, this value is the contents of the `${CI_JOB_TOKEN}` CI/CD variable. + +Outputs from the data source can now be referenced in your Terraform resources +using `data.terraform_remote_state.example.outputs.<OUTPUT-NAME>`. + +To read the Terraform state in the target project, you need at least the Developer role. + ## Manage Terraform state files > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/273592) in GitLab 13.8. diff --git a/doc/user/infrastructure/iac/troubleshooting.md b/doc/user/infrastructure/iac/troubleshooting.md index 5817337223f..e187fe54136 100644 --- a/doc/user/infrastructure/iac/troubleshooting.md +++ b/doc/user/infrastructure/iac/troubleshooting.md @@ -97,6 +97,8 @@ As a result, to create a plan and later use the same plan in another CI job, you `Error: Error acquiring the state lock` errors when using `-backend-config=password=$CI_JOB_TOKEN`. This happens because the value of `$CI_JOB_TOKEN` is only valid for the duration of the current job. +Another possible error message for the same problem could be: `Error: Error loading state: HTTP remote state endpoint requires auth`. + As a workaround, use [http backend configuration variables](https://www.terraform.io/docs/language/settings/backends/http.html#configuration-variables) in your CI job, which is what happens behind the scenes when following the [Get started using GitLab CI](terraform_state.md#initialize-a-terraform-state-as-a-backend-by-using-gitlab-cicd) instructions. diff --git a/doc/user/markdown.md b/doc/user/markdown.md index b1e6fcb2f4e..6a524fe206a 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -190,8 +190,8 @@ end #### PlantUML -To make PlantUML available in GitLab, a GitLab administrator must enable it. For more information, see the -[PlantUML & GitLab](../administration/integration/plantuml.md) page. +PlantUML integration is enabled on GitLab.com. To make PlantUML available in self-managed +installation of GitLab, a GitLab administrator [must enable it](../administration/integration/plantuml.md). #### Kroki @@ -376,6 +376,8 @@ the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activati ### Task lists +> Inapplicable checkboxes [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85982) in GitLab 15.3. + [View this topic in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#task-lists). You can add task lists anywhere Markdown is supported. @@ -384,22 +386,28 @@ You can add task lists anywhere Markdown is supported. - In all other places, you cannot select the boxes. You must edit the Markdown manually by adding or removing an `x` in the brackets. +Besides complete and incomplete, tasks can also be **inapplicable**. Selecting an inapplicable checkbox +in an issue, merge request, or comment has no effect. + To create a task list, follow the format of an ordered or unordered list: ```markdown - [x] Completed task +- [~] Inapplicable task - [ ] Incomplete task - - [ ] Sub-task 1 - - [x] Sub-task 2 + - [x] Sub-task 1 + - [~] Sub-task 2 - [ ] Sub-task 3 1. [x] Completed task +1. [~] Inapplicable task 1. [ ] Incomplete task - 1. [ ] Sub-task 1 - 1. [x] Sub-task 2 + 1. [x] Sub-task 1 + 1. [~] Sub-task 2 + 1. [ ] Sub-task 3 ``` - + ### Table of contents @@ -1398,6 +1406,42 @@ For example: 1. Another item +--- + +Ordered lists that are the first sub-item of an unordered list item must have a preceding blank line if they don't start with `1.`. + +**Good** + +```markdown +- Unordered list item + + 5. First ordered list item +``` + +**Bad** + +```markdown +- Unordered list item + 5. First ordered list item +``` + +--- + +CommonMark ignores blank lines between ordered and unordered list items, and considers them part of a single list. These are rendered as a +_[loose](https://spec.commonmark.org/0.30/#loose)_ list. Each list item is enclosed in a paragraph tag and, therefore, has paragraph spacing and margins. +This makes the list look like there is extra spacing between each item. + +For example: + +```markdown +- First list item +- Second list item + +- A different list +``` + +CommonMark ignores the blank line and renders this as one list with paragraph spacing. + ### Superscripts / Subscripts CommonMark and GitLab Flavored Markdown don't support the Redcarpet superscript syntax ( `x^2` ). diff --git a/doc/user/namespace/index.md b/doc/user/namespace/index.md new file mode 100644 index 00000000000..9ffc65a4e8e --- /dev/null +++ b/doc/user/namespace/index.md @@ -0,0 +1,21 @@ +--- +stage: Manage +group: Workspace +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Namespaces + +In GitLab, a *namespace* organizes related projects together. +GitLab has two types of namespaces: + +- A *personal* namespace, which is based on your username. Projects under a personal namespace must be configured one at a time. +- A *group* or *subgroup* namespace. In these namespaces, you can manage multiple projects at once. + +To determine whether you're viewing a group or personal namespace, you can view the URL. For example: + +| Namespace for | URL | Namespace | +| ------------- | --- | --------- | +| A user named `alex`. | `https://gitlab.example.com/alex` | `alex` | +| A group named `alex-team`. | `https://gitlab.example.com/alex-team` | `alex-team` | +| A group named `alex-team` with a subgroup named `marketing`. | `https://gitlab.example.com/alex-team/marketing` | `alex-team/marketing` | diff --git a/doc/user/packages/composer_repository/index.md b/doc/user/packages/composer_repository/index.md index 901fb740717..4fc55d18253 100644 --- a/doc/user/packages/composer_repository/index.md +++ b/doc/user/packages/composer_repository/index.md @@ -44,7 +44,7 @@ a group's namespace, rather than a user's namespace. Composer packages 1. Run [`composer init`](https://getcomposer.org/doc/03-cli.md#init) and answer the prompts. - For namespace, enter your unique [namespace](../../../user/group/index.md#namespaces), like your GitLab username or group name. + For namespace, enter your unique [namespace](../../../user/namespace/index.md), like your GitLab username or group name. A file called `composer.json` is created: diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index d0c771ecc41..a203de2ed2c 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -555,6 +555,12 @@ this setting. However, disabling the Container Registry disables all Container R ## Troubleshooting the GitLab Container Registry +## Migrating OCI container images to GitLab Container Registry + +Migrating built container images to the GitLab registry is not a current feature. However, an [epic](https://gitlab.com/groups/gitlab-org/-/epics/5210) is open to track the work on this feature. + +Some third-party tools can help migrate container images, for example, [skopeo](https://github.com/containers/skopeo), which can [copy container images](https://github.com/containers/skopeo#copying-images) between various storage mechanisms. You can use skopeo to copy from container registries, container storage backends, local directories, and local OCI-layout directories to the GitLab Container Registry. + ### Docker connection error A Docker connection error can occur when there are special characters in either the group, @@ -563,7 +569,7 @@ project or branch name. Special characters can include: - Leading underscore - Trailing hyphen/dash -To get around this, you can [change the group path](../../group/index.md#change-a-groups-path), +To get around this, you can [change the group path](../../group/manage.md#change-a-groups-path), [change the project path](../../project/settings/index.md#rename-a-repository) or change the branch name. diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 3e41a260854..788e57b6410 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -16,9 +16,26 @@ to automatically manage your container registry usage. ## Check Container Registry Storage Use -The Usage Quotas page (**Settings > Usage Quotas > Storage**) displays storage usage for Packages, -which doesn't include the Container Registry. To track work on this, see the epic -[Storage management for the Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/7226). +The Usage Quotas page (**Settings > Usage Quotas > Storage**) displays storage usage for Packages. +This page includes the Container Registry usage but is currently only available on GitLab.com. +Measuring usage is only possible on the new version of the GitLab Container Registry backed by a +metadata database. We are completing the [upgrade and migration of the GitLab.com Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/5523) +first and only then will work on [making this available to self-managed installs](https://gitlab.com/groups/gitlab-org/-/epics/5521). + +Image layers stored in the Container Registry are deduplicated at the root namespace level. +Therefore, if you tag the same 500MB image twice (either in the same repository or across distinct +repositories under the same root namespace), it will only count towards the root namespace usage +once. Similarly, if a given image layer is shared across multiple images, be those under the same +container repository, project, group, or across different ones, that layer will count only once +towards the root namespace usage. + +Only layers that are referenced by tagged images are accounted for. Untagged images and any layers +referenced exclusively by them are subject to [online garbage collection](index.md#delete-images) +and automatically deleted after 24 hours if they remain unreferenced during that period. + +Image layers are stored on the storage backend in the original (usually compressed) format. This +means that the measured size for any given image layer should match the size displayed on the +corresponding [image manifest](https://github.com/opencontainers/image-spec/blob/main/manifest.md#example-image-manifest). ## Cleanup policy diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index 4770057e4ea..ea9435de12a 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -316,6 +316,28 @@ services: alias: docker ``` +### Issues when authenticating to the Dependency Proxy from CI/CD jobs + +GitLab Runner will automatically authenticate to the Dependency Proxy. However, the underlying Docker engine is still subject to its [authorization resolving process](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/main/docs/configuration/advanced-configuration.md#precedence-of-docker-authorization-resolving). + +Misconfigurations in the authentication mechanism may cause `HTTP Basic: Access denied` and `403: Access forbidden` errors. + +You can use the job logs to view the authentication mechanism used to authenticate against the Dependency Proxy: + +```plaintext +Authenticating with credentials from $DOCKER_AUTH_CONFIG +``` + +```plaintext +Authenticating with credentials from /root/.docker/config.json +``` + +```plaintext +Authenticating with credentials from job payload (GitLab Registry) +``` + +Make sure you are using the expected authentication mechanism. + ### "Not Found" error when pulling image Docker errors similar to the following may indicate that the user running the build job doesn't have diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md index 8d5fc73ad4e..d4acb14b9ca 100644 --- a/doc/user/packages/generic_packages/index.md +++ b/doc/user/packages/generic_packages/index.md @@ -73,6 +73,7 @@ Example request with attribute `select = package_file`: ```shell curl --header "PRIVATE-TOKEN: <your_access_token>" \ + --user "<username>:<Project Access Token>" \ --upload-file path/to/file.txt \ "https://gitlab.example.com/api/v4/projects/24/packages/generic/my_package/0.0.1/file.txt?select=package_file" ``` diff --git a/doc/user/packages/helm_repository/index.md b/doc/user/packages/helm_repository/index.md index 07e853fa18c..f64a269938f 100644 --- a/doc/user/packages/helm_repository/index.md +++ b/doc/user/packages/helm_repository/index.md @@ -71,7 +71,7 @@ Once built, a chart can be uploaded to the desired channel with `curl` or `helm ### Release channels -You can publish Helm charts to channels in GitLab. Channels are a method you can use to differentiate Helm chart repositories. +You can publish Helm charts to channels in GitLab. Channels are a method you can use to differentiate Helm chart repositories. For example, you can use `stable` and `devel` as channels to allow users to add the `stable` repo while `devel` charts are isolated. ## Use CI/CD to publish a Helm package @@ -129,7 +129,7 @@ See [Using Helm](https://helm.sh/docs/intro/using_helm/) for more information. ### The chart is not visible in the Package Registry after uploading -Check the [Sidekiq log](../../../administration/logs.md#sidekiqlog) +Check the [Sidekiq log](../../../administration/logs/index.md#sidekiqlog) for any related errors. If you see `Validation failed: Version is invalid`, it means that the version in your `Chart.yaml` file does not follow [Helm Chart versioning specifications](https://helm.sh/docs/topics/charts/#charts-and-versioning). To fix the error, use the correct version syntax and upload the chart again. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 7ea3c1aa0c8..28de06b2d8a 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -585,7 +585,7 @@ NPM_TOKEN=<your_token> npm install If you get this error, ensure that: - The Package Registry is enabled in your project settings. Although the Package Registry is enabled - by default, it's possible to [disable it](../package_registry/#disable-the-package-registry). + by default, it's possible to [disable it](../package_registry/index.md#disable-the-package-registry). - Your token is not expired and has appropriate permissions. - A package with the same name or version doesn't already exist within the given scope. - The scoped packages URL includes a trailing slash: @@ -634,7 +634,7 @@ This might be accompanied by another error: This is usually a permissions issue with either: - `'packages_storage_path'` default `/var/opt/gitlab/gitlab-rails/shared/packages/`. -- The remote bucket if [object storage](../../../administration/packages/#using-object-storage) +- The remote bucket if [object storage](../../../administration/packages/index.md#using-object-storage) is used. In the latter case, ensure the bucket exists and GitLab has write access to it. diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index d68c7218ca5..7748780d0e4 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -60,11 +60,8 @@ For most package types, the following credential types are valid: allows access to packages in the project running the job for the users running the pipeline. Access to other external projects can be configured. - NOTE: - There's an open issue, - [GitLab-#333444](https://gitlab.com/gitlab-org/gitlab/-/issues/333444), - which prevents you from using a job token with internal projects. This bug only impacts self-managed - GitLab instances. +NOTE: +If you have not activated the "Packages" feature for your project at **Settings > General > Project features**, you will receive a 403 Forbidden response. ## Use GitLab CI/CD to build packages diff --git a/doc/user/packages/package_registry/reduce_package_registry_storage.md b/doc/user/packages/package_registry/reduce_package_registry_storage.md index 4a03bd9e8a0..cd7dd062f60 100644 --- a/doc/user/packages/package_registry/reduce_package_registry_storage.md +++ b/doc/user/packages/package_registry/reduce_package_registry_storage.md @@ -53,7 +53,7 @@ The package files are permanently deleted. ## Cleanup policy -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346153) in GitLab 15.2. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346153) in GitLab 15.2. Depending on the number of packages to remove, the process of manually deleting the packages can take a long time to finish. A cleanup policy defines a set of rules that, applied to a project, defines which package files you can automatically delete. diff --git a/doc/user/packages/terraform_module_registry/index.md b/doc/user/packages/terraform_module_registry/index.md index 2668b8b35ac..436c55f9ee0 100644 --- a/doc/user/packages/terraform_module_registry/index.md +++ b/doc/user/packages/terraform_module_registry/index.md @@ -101,7 +101,7 @@ module "<module>" { } ``` -Where `<namespace>` is the [namespace](../../../user/group/index.md#namespaces) of the Terraform module registry. +Where `<namespace>` is the [namespace](../../../user/namespace/index.md) of the Terraform module registry. ## Publish a Terraform module by using CI/CD @@ -125,7 +125,7 @@ upload: script: - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens - tar -vczf ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . - - 'curl --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" + - 'curl --location --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file ${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/terraform/modules/${TERRAFORM_MODULE_NAME}/${TERRAFORM_MODULE_SYSTEM}/${TERRAFORM_MODULE_VERSION}/file' rules: diff --git a/doc/user/permissions.md b/doc/user/permissions.md index b01bfbef3aa..c89dd3f65f7 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -39,10 +39,10 @@ usernames. A GitLab administrator can configure the GitLab instance to A user's role determines what permissions they have on a project. The Owner role provides all permissions but is available only: -- For group owners. The role is inherited for a group's projects. +- For group and project Owners. In GitLab 14.8 and earlier, the role is inherited for a group's projects. - For Administrators. -Personal [namespace](group/index.md#namespaces) owners: +Personal [namespace](namespace/index.md) owners: - Are displayed as having the Maintainer role on projects in the namespace, but have the same permissions as a user with the Owner role. - In GitLab 14.9 and later, for new projects in the namespace, are displayed as having the Owner role. @@ -95,6 +95,7 @@ The following table lists project permissions available for each role: | [Issues](project/issues/index.md):<br>View [Design Management](project/issues/design_management.md) pages | ✓ | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):<br>View [related issues](project/issues/related_issues.md) | ✓ | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):<br>Set [weight](project/issues/issue_weight.md) | ✓ (*15*) | ✓ | ✓ | ✓ | ✓ | +| [Issues]](project/issues/index.md):<br>Set [parent epic](group/epics/manage_epics.md#add-an-existing-issue-to-an-epic) | | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):<br>View [confidential issues](project/issues/confidential_issues.md) | (*2*) | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):<br>Close / reopen (*19*) | | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):<br>Lock threads | | ✓ | ✓ | ✓ | ✓ | @@ -214,18 +215,18 @@ The following table lists project permissions available for each role: [GitLab.com visibility settings](gitlab_com/index.md#visibility-settings). 2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to. 3. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [protected branches](project/protected_branches.md). -4. If the [branch is protected](project/protected_branches.md), this depends on the access Developers and Maintainers are given. +4. If the [branch is protected](project/protected_branches.md), this depends on the access given to Developers and Maintainers. 5. Guest users can access GitLab [**Releases**](project/releases/index.md) for downloading assets but are not allowed to download the source code nor see [repository information like commits and release evidence](project/releases/index.md#view-a-release-and-download-assets). 6. Actions are limited only to records owned (referenced) by user. -7. When [Share Group Lock](group/index.md#prevent-a-project-from-being-shared-with-groups) is enabled the project can't be shared with other groups. It does not affect group with group sharing. +7. When [Share Group Lock](group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups) is enabled the project can't be shared with other groups. It does not affect group with group sharing. 8. For information on eligible approvers for merge requests, see [Eligible approvers](project/merge_requests/approvals/rules.md#eligible-approvers). 9. Applies only to comments on [Design Management](project/issues/design_management.md) designs. 10. Users can only view events based on their individual actions. 11. Project access tokens are supported for self-managed instances on Free and above. They are also supported on GitLab SaaS Premium and above (excluding [trial licenses](https://about.gitlab.com/free-trial/)). -12. If the [tag is protected](#release-permissions-with-protected-tags), this depends on the access Developers and Maintainers are given. -13. A Maintainer can't change project features visibility level if +12. If the [tag is protected](#release-permissions-with-protected-tags), this depends on the access given to Developers and Maintainers. +13. A Maintainer or Owner can't change project features visibility level if [project visibility](public_access.md) is set to private. 14. Attached design files are moved together with the issue even if the user doesn't have the Developer role. @@ -333,10 +334,9 @@ which visibility level you select on project settings. ### Protected branches Additional restrictions can be applied on a per-branch basis with [protected branches](project/protected_branches.md). -Additionally, you can customize permissions to allow or prevent project -Maintainers and Developers from pushing to a protected branch. Read through the documentation on -[protected branches](project/protected_branches.md) -to learn more. +Additionally, you can customize permissions to allow or prevent project Developers or Maintainers +from pushing to a protected branch. Read through the documentation on +[protected branches](project/protected_branches.md) to learn more. ### Value stream analytics permissions @@ -412,7 +412,7 @@ The following table lists group permissions available for each role: | Delete [group wiki](project/wiki/group.md) pages | | | ✓ | ✓ | ✓ | | Edit [epic](group/epics/index.md) comments (posted by any user) | | | | ✓ (2) | ✓ (2) | | List group deploy tokens | | | | ✓ | ✓ | -| Manage [group push rules](group/index.md#group-push-rules) | | | | ✓ | ✓ | +| Manage [group push rules](group/access_and_permissions.md#group-push-rules) | | | | ✓ | ✓ | | View/manage group-level Kubernetes cluster | | | | ✓ | ✓ | | Create and manage compliance frameworks | | | | | ✓ | | Create/Delete group deploy tokens | | | | | ✓ | @@ -439,9 +439,9 @@ The following table lists group permissions available for each role: 2. Introduced in GitLab 12.2. 3. Default project creation role can be changed at: - The [instance level](admin_area/settings/visibility_and_access_controls.md#define-which-roles-can-create-projects). - - The [group level](group/index.md#specify-who-can-add-projects-to-a-group). + - The [group level](group/manage.md#specify-who-can-add-projects-to-a-group). 4. Does not apply to subgroups. -5. Developers can push commits to the default branch of a new project only if the [default branch protection](group/index.md#change-the-default-branch-protection-of-a-group) is set to "Partially protected" or "Not protected". +5. Developers can push commits to the default branch of a new project only if the [default branch protection](group/manage.md#change-the-default-branch-protection-of-a-group) is set to "Partially protected" or "Not protected". 6. In addition, if your group is public or internal, all users who can see the group can also see group wiki pages. 7. Users can only view events based on their individual actions. @@ -600,7 +600,7 @@ for more information. ## LDAP users permissions LDAP user permissions can be manually overridden by an administrator. -Read through the documentation on [LDAP users permissions](group/index.md#manage-group-memberships-via-ldap) to learn more. +Read through the documentation on [LDAP users permissions](group/access_and_permissions.md#manage-group-memberships-via-ldap) to learn more. ## Project aliases diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index 3f5554b80ac..5ec29814e06 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -78,7 +78,7 @@ in users not being deleted, and the following error generated: ERROR: null value in column "user_id" violates not-null constraint ``` -The error can be found in the [PostgreSQL log](../../../administration/logs.md#postgresql-logs) and +The error can be found in the [PostgreSQL log](../../../administration/logs/index.md#postgresql-logs) and in the **Retries** section of the [background jobs view](../../admin_area/index.md#background-jobs) in the Admin Area. If the user being deleted used the [iterations](../../group/iterations/index.md) feature, such diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index 4563cfe5648..3af033c7130 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -59,15 +59,12 @@ To enable 2FA with a one-time password: 1. Select **Enable Two-factor Authentication**. 1. **On your device (usually your phone):** 1. Install a compatible application. For example: - - [Aegis](https://getaegis.app/) - - [Raivo OTP](https://apps.apple.com/us/app/raivo-otp/id1459042137#platform=iphone) - - [Authy](https://authy.com/) - - [Duo Mobile](https://duo.com/product/multi-factor-authentication-mfa/duo-mobile-app) - - [LastPass Authenticator](https://lastpass.com/auth/) - - [Authenticator](https://mattrubin.me/authenticator/) - - [Google Authenticator](https://support.google.com/accounts/answer/1066447?hl=en) - - [Microsoft Authenticator](https://www.microsoft.com/en-us/security/mobile-authenticator-app) - - [SailOTP](https://openrepos.net/content/seiichiro0185/sailotp) + - Cloud-based (recommended because you can restore access if you lose the hardware device): + - [Authy](https://authy.com/) + - [Duo Mobile](https://duo.com/product/multi-factor-authentication-mfa/duo-mobile-app) + - Other: + - [Google Authenticator](https://support.google.com/accounts/answer/1066447?hl=en) + - [Microsoft Authenticator](https://www.microsoft.com/en-us/security/mobile-authenticator-app) 1. In the application, add a new entry in one of two ways: - Scan the code displayed by GitLab with your device's camera to add the entry automatically. - Enter the details provided to add the entry manually. @@ -358,8 +355,7 @@ After you use a recovery code, you cannot re-use it. You can still use the other ### Generate new recovery codes using SSH -Users often forget to save their recovery codes when enabling 2FA. If you added an SSH key to your -GitLab account, you can generate a new set of recovery codes with SSH: +If you forget to save your recovery codes when enabling 2FA, and you added an SSH key to your GitLab account, you can generate a new set of recovery codes with SSH: 1. In a terminal, run: @@ -403,10 +399,9 @@ After signing in, immediately set up 2FA with a new device. ### Have two-factor authentication disabled on your account **(PREMIUM SAAS)** -If other methods are unavailable, submit a [support ticket](https://support.gitlab.com/hc/en-us/requests/new) to request +If other methods are unavailable, have a GitLab support contact submit a [support ticket](https://support.gitlab.com) to request a GitLab global administrator disable 2FA for your account: -- Only the owner of the account can make this request. - This service is only available for accounts that have a GitLab.com subscription. For more information, see our [blog post](https://about.gitlab.com/blog/2020/08/04/gitlab-support-no-longer-processing-mfa-resets-for-free-users/). - Disabling this setting temporarily leaves your account in a less secure state. You should sign in and re-enable two-factor diff --git a/doc/user/profile/index.md b/doc/user/profile/index.md index bf696310158..4c859d98004 100644 --- a/doc/user/profile/index.md +++ b/doc/user/profile/index.md @@ -36,11 +36,11 @@ To change your password: 1. In the **New password** and **Password confirmation** text box, enter your new password. 1. Select **Save password**. -If you don't know your current password, select the **I forgot my password** link. +If you don't know your current password, select the **I forgot my password** link. A password reset email is sent to the account's **primary** email address. ## Change your username -Your username has a unique [namespace](../group/index.md#namespaces), +Your username has a unique [namespace](../namespace/index.md), which is updated when you change your username. Before you change your username, read about [how redirects behave](../project/repository/index.md#what-happens-when-a-repository-path-changes). If you do not want to update the namespace, you can create a new user or group and transfer projects to it instead. diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index 3eda363d108..0245f0615e0 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -7,9 +7,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Notification emails **(FREE)** -> Enhanced email styling [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78604) in GitLab 14.9 [with a feature flag](../../administration/feature_flags.md) named `enhanced_notify_css`. Disabled by default. -> Enhanced email styling [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 14.9. -> Enhanced email styling [enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 15.0. +> - Enhanced email styling [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78604) in GitLab 14.9 [with a feature flag](../../administration/feature_flags.md) named `enhanced_notify_css`. Disabled by default. +> - Enhanced email styling [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 14.9. +> - Enhanced email styling [enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/355907) in GitLab 15.0. Stay informed about what's happening in GitLab with email notifications. You can receive updates about activity in issues, merge requests, epics, and designs. diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index a76517a7341..f8494116655 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -38,9 +38,9 @@ want to make sure the performance stays the same, or improves. Developers need to be careful when using canaries with user-facing changes, because by default, requests from the same user are randomly distributed between canary and non-canary pods, which could result in confusion or even errors. If needed, you -may want to consider [setting `service.spec.sessionAffinity` to `ClientIP` in -your Kubernetes service definitions](https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies), but that is beyond the scope of -this document. +may want to consider +[setting `service.spec.sessionAffinity` to `ClientIP` in your Kubernetes service definitions](https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies), +but that is beyond the scope of this document. ## Advanced traffic control with Canary Ingress diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index 2e1c8766ae3..bfaf9aab7b7 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -37,7 +37,7 @@ Prerequisites: set up with access. - Kubernetes Engine API and related services enabled. It should work immediately but may take up to 10 minutes after you create a project. For more information see the - ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin). + ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/deploy-app-cluster#before-you-begin). Note the following: @@ -51,8 +51,8 @@ Note the following: cluster's pod address IP range is set to `/16` instead of the regular `/14`. `/16` is a CIDR notation. - GitLab requires basic authentication enabled and a client certificate issued for the cluster to - set up an [initial service account](cluster_access.md). In [GitLab versions - 11.10 and later](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58208), the cluster creation process + set up an [initial service account](cluster_access.md). In + [GitLab versions 11.10 and later](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/58208), the cluster creation process explicitly requests GKE to create clusters with basic authentication enabled and a client certificate. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index 95d8064b380..f1004a40a13 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -46,3 +46,27 @@ To remove the Kubernetes cluster integration: 1. Go to your cluster details page. 1. Select the **Advanced Settings** tab. 1. Select either **Remove integration** or **Remove integration and resources**. + +### Remove clusters by using the Rails console **(FREE SELF)** + +[Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session). + +To find a cluster: + +``` ruby +cluster = Clusters::Cluster.find(1) +cluster = Clusters::Cluster.find_by(name: 'cluster_name') +``` + +To delete a cluster but not the associated resources: + +```ruby +# Find users who have administrator access +user = User.find_by(username: 'admin_user') + +# Find the cluster with the ID +cluster = Clusters::Cluster.find(1) + +# Delete the cluster +Clusters::DestroyService.new(user).execute(cluster) +``` diff --git a/doc/user/project/clusters/deploy_to_cluster.md b/doc/user/project/clusters/deploy_to_cluster.md index fc41533b17c..0a574b9afe2 100644 --- a/doc/user/project/clusters/deploy_to_cluster.md +++ b/doc/user/project/clusters/deploy_to_cluster.md @@ -131,7 +131,7 @@ However, sometimes GitLab cannot create them. In such instances, your job can fa This job failed because the necessary resources were not successfully created. ``` -To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog). +To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs/index.md#kuberneteslog). Reasons for failure include: diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index adea5dad7b8..fd2df96308c 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -8,21 +8,37 @@ info: To determine the technical writer assigned to the Stage/Group associated w > Moved to GitLab Premium in 13.9. -Code Owners define who owns specific files or directories in a repository. +Code Owners define who develops and maintains a feature, and own the resulting +files or directories in a repository. - The users you define as Code Owners are displayed in the UI when you browse directories. - You can set your merge requests so they must be approved by Code Owners before merge. - You can protect a branch and allow only Code Owners to approve changes to the branch. -If you don't want to use Code Owners for approvals, you can -[configure rules](merge_requests/approvals/rules.md) instead. +Use Code Owners and approvers together with +[approval rules](merge_requests/approvals/rules.md) to build a flexible approval +workflow: -## Set up Code Owners +- Use **Code Owners** to define the users who have domain expertise for specific paths in your repository. +- Use **Approvers** and **Approval rules** to define domains of expertise (such as a security team) + that are not scoped to specific file paths in your repository. + - **Approvers** define the users. + - **Approval rules** define when these users can approve work, and whether or not their approval is required. + +For example: + +| Type | Name | Scope | Comment | +|------|------|--------|------------| +| Approval rule | UX | All files | A user experience (UX) team member reviews the user experience of all changes made in your project. | +| Approval rule | Security | All files | A security team member reviews all changes for vulnerabilities. | +| Code Owner approval rule | Frontend: Code Style | `*.css` files | A frontend engineer reviews CSS file changes for adherence to project style standards. | +| Code Owner approval rule | Backend: Code Review | `*.rb` files | A backend engineer reviews the logic and code style of Ruby files. | -You can use Code Owners to specify users or [shared groups](members/share_project_with_groups.md) -that are responsible for specific files and directories in a repository. +## Set up Code Owners -To set up Code Owners: +Create a `CODEOWNERS` file to specify users or [shared groups](members/share_project_with_groups.md) +that are responsible for specific files and directories in a repository. Each repository +can have a single `CODEOWNERS` file. To create it: 1. Choose the location where you want to specify Code Owners: - In the root directory of the repository @@ -59,24 +75,38 @@ Next steps: > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) in GitLab 12.1. > - Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. -You can use members of groups and subgroups as Code Owners for a project. +You can use members of groups and subgroups as Code Owners for projects: -For example, if you have these groups: +```mermaid +graph TD + A[Parent group X] -->|owns| B[Project A] + A -->|contains| C[Subgroup Y] + C -->|owns| D[Project B] + A-. inherits ownership .-> D +``` -- **Group X** (`group-x`) with **Project A** in it. -- **Subgroup Y** (`group-x/subgroup-y`), which belongs to **Group X**, with **Project B** in it. +In this example: -The eligible Code Owners: +- **Parent group X** (`group-x`) owns **Project A**. +- **Parent group X** also contains a subgroup, **Subgroup Y**. (`group-x/subgroup-y`) +- **Subgroup Y** owns **Project B**. -- For **Project A** are the members of **Group X** only, because **Project A** doesn't belong to **Subgroup Y**. -- For **Project B** are the members of both **Group X** and **Subgroup Y**. +The eligible Code Owners are: - +- **Project A**: the members of **Group X** only, because **Project A** doesn't belong to **Subgroup Y**. +- **Project B**: the members of both **Group X** and **Subgroup Y**. You can [invite](members/share_project_with_groups.md) **Subgroup Y** to **Project A** so that their members also become eligible Code Owners. - +```mermaid +graph LR + A[Parent group X] -->|owns| B[Project A] + A -->|also contains| C[Subgroup Y] + C -.->D{Invite Subgroup Y<br/>to Project A?} -.->|yes| F[Approvals can be<br/> required] -.-> B + D{Invite Subgroup Y<br/>to Project A?} -.->|no| I[Subgroup Y cannot be<br /> an approver] -.-> B + C -.->E{Add Subgroup Y<br/>as Code Owners<br/>to Project A?} -.->|yes| H[Approvals can only<br/>be optional] -.-> B +``` If you do not invite **Subgroup Y** to **Project A**, but make them Code Owners, their approval of the merge request becomes optional. diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index a90ffbe5796..41afbdada6b 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -116,8 +116,8 @@ To display the deploy boards for a specific [environment](../../ci/environments/ Kubernetes. NOTE: - Matching based on the Kubernetes `app` label was removed in [GitLab - 12.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/14020). + Matching based on the Kubernetes `app` label was removed in + [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/14020). To migrate, please apply the required annotations (see above) and re-deploy your application. If you are using Auto DevOps, this will be done automatically and no action is necessary. diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md index c64afd95d8d..a9f19d27416 100644 --- a/doc/user/project/deploy_keys/index.md +++ b/doc/user/project/deploy_keys/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Use deploy keys to access repositories that are hosted in GitLab. In most cases, you use deploy keys to access a repository from an external host, like a build server or Continuous Integration (CI) server. -Depending on your needs, you might want to use a [deploy token](../deploy_tokens/) to access a repository instead. +Depending on your needs, you might want to use a [deploy token](../deploy_tokens/index.md) to access a repository instead. | Attribute | Deploy key | Deploy token | |------------------|-------------|--------------| @@ -146,8 +146,8 @@ What happens to the deploy key when it is disabled depends on the following: ### Deploy key cannot push to a protected branch -There are a few scenarios where a deploy key will fail to push to a [protected -branch](../protected_branches.md). +There are a few scenarios where a deploy key will fail to push to a +[protected branch](../protected_branches.md). - The owner associated to a deploy key does not have access to the protected branch. - The owner associated to a deploy key does not have [membership](../members/index.md) to the project of the protected branch. diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index 42287ff84ce..5df3a973cca 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -112,7 +112,7 @@ To re-use templates [you've created](../project/description_templates.md#create-  You might also be interested in templates for various -[file types in groups](../group/index.md#group-file-templates). +[file types in groups](../group/manage.md#group-file-templates). ### Set a default template for merge requests and issues diff --git a/doc/user/project/git_attributes.md b/doc/user/project/git_attributes.md index 11d1db195e1..90f64b7262c 100644 --- a/doc/user/project/git_attributes.md +++ b/doc/user/project/git_attributes.md @@ -23,5 +23,5 @@ ignored. ## Syntax Highlighting The `.gitattributes` file can be used to define which language to use when -syntax highlighting files and diffs. See ["Syntax -Highlighting"](highlighting.md) for more information. +syntax highlighting files and diffs. See +["Syntax Highlighting"](highlighting.md) for more information. diff --git a/doc/user/project/img/code_owners_invite_members_v13_4.png b/doc/user/project/img/code_owners_invite_members_v13_4.png Binary files differdeleted file mode 100644 index 852a5f68b36..00000000000 --- a/doc/user/project/img/code_owners_invite_members_v13_4.png +++ /dev/null diff --git a/doc/user/project/img/code_owners_members_v13_4.png b/doc/user/project/img/code_owners_members_v13_4.png Binary files differdeleted file mode 100644 index e37fe72cd6e..00000000000 --- a/doc/user/project/img/code_owners_members_v13_4.png +++ /dev/null diff --git a/doc/user/project/import/clearcase.md b/doc/user/project/import/clearcase.md index 867477c83b2..d9ad0c57d79 100644 --- a/doc/user/project/import/clearcase.md +++ b/doc/user/project/import/clearcase.md @@ -11,8 +11,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w tools developed by IBM which also include a centralized version control system similar to Git. -A good read of ClearCase's basic concepts is can be found in this [StackOverflow -post](https://stackoverflow.com/a/645771/974710). +A good read of ClearCase's basic concepts is can be found in this +[StackOverflow post](https://stackoverflow.com/a/645771/974710). The following table illustrates the main differences between ClearCase and Git: diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index a190edb179d..a3dfa3edff0 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -241,3 +241,35 @@ Feature.disable(:github_importer_lower_per_page_limit, group) For information on automating user, group, and project import API calls, see [Automate group and project import](index.md#automate-group-and-project-import). + +## Troubleshooting + +### Manually continue a previously failed import process + +In some cases, the GitHub import process can fail to import the repository. This causes GitLab to abort the project import process and requires the +repository to be imported manually. Administrators can manually import the repository for a failed import process: + +1. Open a Rails console. +1. Run the following series of commands in the console: + + ```ruby + project_id = <PROJECT_ID> + github_access_token = <GITHUB_ACCESS_TOKEN> + github_repository_path = '<GROUP>/<REPOSITORY>' + + github_repository_url = "https://#{github_access_token}@github.com/#{github_repository_path}.git" + + # Find project by ID + project = Project.find(project_id) + # Set import URL and credentials + project.import_url = github_repository_url + project.import_type = 'github' + project.import_source = github_repository_path + project.save! + # Create an import state if the project was created manually and not from a failed import + project.create_import_state if project.import_state.blank? + # Set state to start + project.import_state.force_start + # Trigger import from second step + Gitlab::GithubImport::Stage::ImportRepositoryWorker.perform_async(project.id) + ``` diff --git a/doc/user/project/insights/index.md b/doc/user/project/insights/index.md index 0609b843e86..53547e69e00 100644 --- a/doc/user/project/insights/index.md +++ b/doc/user/project/insights/index.md @@ -68,12 +68,14 @@ bugsCharts: description: "Open bugs created per month" type: bar query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - group_by: month - period_limit: 24 + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + group_by: month + period_limit: 24 ``` Each chart definition is made up of a hash composed of key-value pairs. @@ -85,12 +87,14 @@ For example, here's single chart definition: description: "Open bugs created per month" type: bar query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - group_by: month - period_limit: 24 + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + group_by: month + period_limit: 24 ``` ## Configuration parameters @@ -104,7 +108,7 @@ The following table lists available parameters for charts: | [`title`](#title) | The title of the chart. This displays on the Insights page. | | [`description`](#description) | A description for the individual chart. This displays above the relevant chart. | | [`type`](#type) | The type of chart: `bar`, `line` or `stacked-bar`. | -| [`query`](#query) | A hash that defines the conditions for issues / merge requests to be part of the chart. | +| [`query`](#query) | A hash that defines the data source and filtering conditions for the chart. | ## Parameter details @@ -153,10 +157,12 @@ Supported values are: | `line` |  | | `stacked-bar` |  | +NOTE: +The `dora` data source only supports the `bar` chart type. + ### `query` -`query` allows to define the conditions for issues / merge requests to be part -of the chart. +`query` allows to define the data source and various filtering conditions for the chart. Example: @@ -166,6 +172,29 @@ monthlyBugsCreated: description: "Open bugs created per month" type: bar query: + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + collection_labels: + - S1 + - S2 + - S3 + - S4 + group_by: week + period_limit: 104 +``` + +The legacy format without the `data_source` parameter is still supported: + +```yaml +monthlyBugsCreated: + title: "Monthly bugs created" + description: "Open bugs created per month" + type: bar + query: issuable_type: issue issuable_state: opened filter_labels: @@ -179,7 +208,20 @@ monthlyBugsCreated: period_limit: 104 ``` -#### `query.issuable_type` +#### `query.data_source` + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/725) in GitLab 15.3. + +The `data_source` parameter was introduced to allow visualizing data from different data sources. + +Supported values are: + +- `issuables`: Exposes merge request or issue data. +- `dora`: Exposes DORA metrics data. + +#### `Issuable` query parameters + +##### `query.params.issuable_type` Defines the type of "issuable" you want to create a chart for. @@ -188,7 +230,7 @@ Supported values are: - `issue`: The chart displays issues' data. - `merge_request`: The chart displays merge requests' data. -#### `query.issuable_state` +##### `query.params.issuable_state` Filter by the current state of the queried "issuable". @@ -202,7 +244,7 @@ Supported values are: - `merged`: Merged merge requests. - `all`: Issues / merge requests in all states -#### `query.filter_labels` +##### `query.params.filter_labels` Filter by labels currently applied to the queried "issuable". @@ -216,14 +258,16 @@ monthlyBugsCreated: title: "Monthly regressions created" type: bar query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - - regression + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + - regression ``` -#### `query.collection_labels` +##### `query.params.collection_labels` Group "issuable" by the configured labels. @@ -237,18 +281,20 @@ weeklyBugsBySeverity: title: "Weekly bugs by severity" type: stacked-bar query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - collection_labels: - - S1 - - S2 - - S3 - - S4 + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + collection_labels: + - S1 + - S2 + - S3 + - S4 ``` -#### `query.group_by` +##### `query.group_by` Define the X-axis of your chart. @@ -258,7 +304,7 @@ Supported values are: - `week`: Group data per week. - `month`: Group data per month. -#### `query.period_limit` +##### `query.period_limit` Define how far "issuables" are queried in the past (using the `query.period_field`). @@ -295,6 +341,68 @@ NOTE: Until [this bug](https://gitlab.com/gitlab-org/gitlab/-/issues/26911), is resolved, you may see `created_at` in place of `merged_at`. `created_at` is used instead. +#### `DORA` query parameters + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367248) in GitLab 15.3. + +An example DORA chart definition: + +```yaml +dora: + title: "DORA charts" + charts: + - title: "DORA deployment frequency" + type: bar # only bar chart is supported at the moment + query: + data_source: dora + params: + metric: deployment_frequency + group_by: day + period_limit: 10 + projects: + only: + - 38 + - title: "DORA lead time for changes" + description: "DORA lead time for changes" + type: bar + query: + data_source: dora + params: + metric: lead_time_for_changes + group_by: day + environment_tiers: + - staging + period_limit: 30 +``` + +##### `query.metric` + +Defines which DORA metric to query. The available values are: + +- `deployment_frequency` (default) +- `lead_time_for_changes` +- `time_to_restore_service` +- `change_failure_rate` + +The metrics are described on the [DORA API](../../../api/dora/metrics.md#the-value-field) page. + +##### `query.group_by` + +Define the X-axis of your chart. + +Supported values are: + +- `day` (default): Group data per day. +- `month`: Group data per month. + +##### `query.period_limit` + +Define how far the metrics are queried in the past (default: 15). Maximum lookback period is 180 days or 6 months. + +##### `query.environment_tiers` + +An array of environments to include into the calculation (default: production). Available options: `production`, `staging`, `testing`, `development`, `other`. + ### `projects` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10904) in GitLab 12.4. @@ -325,10 +433,12 @@ monthlyBugsCreated: description: "Open bugs created per month" type: bar query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug projects: only: - 3 # You can use the project ID @@ -355,41 +465,47 @@ bugsCharts: type: bar <<: *projectsOnly query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - group_by: month - period_limit: 24 + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + group_by: month + period_limit: 24 - title: "Weekly bugs by severity" type: stacked-bar <<: *projectsOnly query: - issuable_type: issue - issuable_state: opened - filter_labels: - - bug - collection_labels: - - S1 - - S2 - - S3 - - S4 - group_by: week - period_limit: 104 + data_source: issuables + params: + issuable_type: issue + issuable_state: opened + filter_labels: + - bug + collection_labels: + - S1 + - S2 + - S3 + - S4 + group_by: week + period_limit: 104 - title: "Monthly bugs by team" type: line <<: *projectsOnly query: - issuable_type: merge_request - issuable_state: opened - filter_labels: - - bug - collection_labels: - - Manage - - Plan - - Create - group_by: month - period_limit: 24 + data_source: issuables + params: + issuable_type: merge_request + issuable_state: opened + filter_labels: + - bug + collection_labels: + - Manage + - Plan + - Create + group_by: month + period_limit: 24 ``` diff --git a/doc/user/project/integrations/harbor.md b/doc/user/project/integrations/harbor.md index 1319c9e74cd..da35f0dc226 100644 --- a/doc/user/project/integrations/harbor.md +++ b/doc/user/project/integrations/harbor.md @@ -39,7 +39,7 @@ GitLab supports integrating Harbor projects at the group or project level. Compl After the Harbor integration is activated: -- The global variables `$HARBOR_USERNAME`, `$HARBOR_PASSWORD`, `$HARBOR_URL`, and `$HARBOR_PROJECT` are created for CI/CD use. +- The global variables `$HARBOR_USERNAME`, `$HARBOR_HOST`, `$HARBOR_OCI`, `$HARBOR_PASSWORD`, `$HARBOR_URL`, and `$HARBOR_PROJECT` are created for CI/CD use. - The project-level integration settings override the group-level integration settings. ## Secure your requests to the Harbor APIs @@ -50,3 +50,51 @@ the `username:password` combination. The following are suggestions for safe use: - Use TLS on the Harbor APIs you connect to. - Follow the principle of least privilege (for access on Harbor) with your credentials. - Have a rotation policy on your credentials. + +## Examples of Harbor variables in CI/CD + +### Push a Docker image with kaniko + +For more information, see [Use kaniko to build Docker images](../../../ci/docker/using_kaniko.md). + +```yaml +docker: + stage: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [''] + script: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${HARBOR_HOST}\":{\"auth\":\"$(echo -n ${HARBOR_USERNAME}:${HARBOR_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json + - >- + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${HARBOR_HOST}/${HARBOR_PROJECT}/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}" + rules: + - if: $CI_COMMIT_TAG +``` + +### Push a Helm chart with an OCI registry + +Helm supports OCI registries by default. OCI is supported in [Harbor 2.0](https://github.com/goharbor/harbor/releases/tag/v2.0.0) and later. +Read more about OCI in Helm's [blog](https://helm.sh/blog/storing-charts-in-oci/) and [documentation](https://helm.sh/docs/topics/registries/#enabling-oci-support). + +```yaml +helm: + stage: helm + image: + name: dtzar/helm-kubectl:latest + entrypoint: [''] + variables: + # Enable OCI support (not required since Helm v3.8.0) + HELM_EXPERIMENTAL_OCI: 1 + script: + # Log in to the Helm registry + - helm registry login "${HARBOR_URL}" -u "${HARBOR_USERNAME}" -p "${HARBOR_PASSWORD}" + # Package your Helm chart, which is in the `test` directory + - helm package test + # Your helm chart is created with <chart name>-<chart release>.tgz + # You can push all building charts to your Harbor repository + - helm push test-*.tgz ${HARBOR_OCI}/${HARBOR_PROJECT} +``` diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 7af2e431157..3ef40fdfe44 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -73,6 +73,7 @@ You can configure the following integrations. | [Pipelines emails](pipeline_status_emails.md) | Send the pipeline status to a list of recipients by email. | **{dotted-circle}** No | | [Pivotal Tracker](pivotal_tracker.md) | Add commit messages as comments to Pivotal Tracker stories. | **{dotted-circle}** No | | [Prometheus](prometheus.md) | Monitor application metrics. | **{dotted-circle}** No | +| [Pumble](pumble.md) | Send event notifications to a Pumble channel. | **{dotted-circle}** No | | Pushover | Get real-time notifications on your device. | **{dotted-circle}** No | | [Redmine](redmine.md) | Use Redmine as the issue tracker. | **{dotted-circle}** No | | [Slack application](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | diff --git a/doc/user/project/integrations/pumble.md b/doc/user/project/integrations/pumble.md new file mode 100644 index 00000000000..cd28a7c0048 --- /dev/null +++ b/doc/user/project/integrations/pumble.md @@ -0,0 +1,39 @@ +--- +stage: Ecosystem +group: Integrations +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Pumble **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93623) in GitLab 15.3. + +You can configure GitLab to send notifications to a Pumble channel: + +1. Create a webhook for the channel. +1. Add the webhook to GitLab. + +## Create a webhook for your Pumble channel + +1. Follow the steps in [Incoming Webhooks for Pumble](https://pumble.com/help/integrations/custom-apps/incoming-webhooks-for-pumble/) in the Pumble documentation. +1. Copy the webhook URL. + +## Configure settings in GitLab + +After you have a webhook URL for your Pumble channel, configure GitLab to send +notifications: + +1. To enable the integration for your group or project: + 1. In your group or project, on the left sidebar, select **Settings > Integrations**. +1. To enable the integration for your instance: + 1. On the top bar, select **Menu > Admin**. + 1. On the left sidebar, select **Settings > Integrations**. +1. Select the **Pumble** integration. +1. Ensure that the **Active** toggle is enabled. +1. Select the checkboxes corresponding to the GitLab events you want to receive in Pumble. +1. Paste the **Webhook** URL for the Pumble channel. +1. Configure the remaining options. +1. Optional. To test the integration, select **Test settings**. +1. Select **Save changes**. + +The Pumble channel begins to receive all applicable GitLab events. diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index bd93fbcbcbc..dd0f57570aa 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -77,13 +77,13 @@ The following triggers are available for Slack notifications: ## Troubleshooting If your Slack integration is not working, start troubleshooting by -searching through the [Sidekiq logs](../../../administration/logs.md#sidekiqlog) +searching through the [Sidekiq logs](../../../administration/logs/index.md#sidekiqlog) for errors relating to your Slack service. ### Something went wrong on our end You might get this generic error message in the GitLab UI. -Review [the logs](../../../administration/logs.md#productionlog) to find +Review [the logs](../../../administration/logs/index.md#productionlog) to find the error message and keep troubleshooting from there. ### `certificate verify failed` diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index 32e5f949c15..51049f156b8 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -878,7 +878,9 @@ Payload example: "source_branch": "ms-viewport", "source_project_id": 14, "author_id": 51, + "assignee_ids": [6], "assignee_id": 6, + "reviewer_ids": [6], "title": "MS-Viewport", "created_at": "2013-12-03T17:23:34Z", "updated_at": "2013-12-03T17:23:34Z", @@ -945,12 +947,7 @@ Payload example: "type": "ProjectLabel", "group_id": 41 }], - "action": "open", - "assignee": { - "name": "User1", - "username": "user1", - "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" - } + "action": "open" }, "labels": [{ "id": 206, @@ -999,7 +996,23 @@ Payload example: "group_id": 41 }] } - } + }, + "assignees": [ + { + "id": 6, + "name": "User1", + "username": "user1", + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" + } + ], + "reviewers": [ + { + "id": 6, + "name": "User1", + "username": "user1", + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40\u0026d=identicon" + } + ] } ``` @@ -1614,7 +1627,7 @@ Payload example: ### Remove a subgroup from a group -This webhook is not triggered when a [subgroup is transferred to a new parent group](../../group/index.md#transfer-a-group). +This webhook is not triggered when a [subgroup is transferred to a new parent group](../../group/manage.md#transfer-a-group). Request header: diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index e4ce736684b..1de7440a15d 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -137,7 +137,7 @@ For example, to test `push events`, your project should have at least one commit To test a webhook: -1. In your project, on the left sidebar, select **Settings > Webhooks**. +1. In your project or group, on the left sidebar, select **Settings > Webhooks**. 1. Scroll down to the list of configured webhooks. 1. From the **Test** dropdown list, select the type of event to test. @@ -236,12 +236,14 @@ For more information about supported events for Webhooks, go to [Webhook events] ## Troubleshoot webhooks +> **Recent events** for group webhooks [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325642) in GitLab 15.3. + GitLab records the history of each webhook request. You can view requests made in the last 2 days in the **Recent events** table. To view the table: -1. In your project, on the left sidebar, select **Settings > Webhooks**. +1. In your project or group, on the left sidebar, select **Settings > Webhooks**. 1. Scroll down to the webhooks. 1. Each [failing webhook](#failing-webhooks) has a badge listing it as: @@ -261,10 +263,6 @@ The table includes the following details about each request:  -NOTE: -The **Recent events** table is unavailable for group-level webhooks. For more information, read -[issue #325642](https://gitlab.com/gitlab-org/gitlab/-/issues/325642). - Each webhook event has a corresponding **Details** page. This page details the data that GitLab sent (request headers and body) and received (response headers and body). To view the **Details** page, select **View details** for the webhook event. diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index c8ecb2fd2e6..787e990526d 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -642,3 +642,20 @@ A few things to remember: - For performance and visibility reasons, each list shows the first 20 issues by default. If you have more than 20 issues, start scrolling down and the next 20 appear. + +## Troubleshooting issue boards + +### Use Rails console to fix issue boards not loading and timing out + +If you see issue board not loading and timing out in UI, use Rails console to call the Issue Rebalancing service to fix it: + +1. [Start a Rails console session](../../administration/operations/rails_console.md#starting-a-rails-console-session). +1. Run these commands: + + ```ruby + p = Project.find_by_full_path('<username-or-group>/<project-name>') + + Issues::RelativePositionRebalancingService.new(p.root_namespace.all_projects).execute + ``` + +1. To exit the Rails console, type `quit`. diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index d1b27f6eab0..bf3cd13f3f0 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -276,7 +276,7 @@ It's rendered as: User activity events on designs (creation, deletion, and updates) are tracked by GitLab and displayed on the [user profile](../../profile/index.md#access-your-user-profile), -[group](../../group/index.md#view-group-activity), +[group](../../group/manage.md#view-group-activity), and [project](../working_with_projects.md#view-project-activity) activity pages. ## GitLab-Figma plugin diff --git a/doc/user/project/issues/img/related_issue_block_v12_8.png b/doc/user/project/issues/img/related_issue_block_v12_8.png Binary files differdeleted file mode 100644 index ce261f26ce6..00000000000 --- a/doc/user/project/issues/img/related_issue_block_v12_8.png +++ /dev/null diff --git a/doc/user/project/issues/img/related_issue_block_v15_3.png b/doc/user/project/issues/img/related_issue_block_v15_3.png Binary files differnew file mode 100644 index 00000000000..827ddeabf10 --- /dev/null +++ b/doc/user/project/issues/img/related_issue_block_v15_3.png diff --git a/doc/user/project/issues/img/related_issues_add_v12_8.png b/doc/user/project/issues/img/related_issues_add_v12_8.png Binary files differdeleted file mode 100644 index 8a06d005a5f..00000000000 --- a/doc/user/project/issues/img/related_issues_add_v12_8.png +++ /dev/null diff --git a/doc/user/project/issues/img/related_issues_add_v15_3.png b/doc/user/project/issues/img/related_issues_add_v15_3.png Binary files differnew file mode 100644 index 00000000000..7c6edf61427 --- /dev/null +++ b/doc/user/project/issues/img/related_issues_add_v15_3.png diff --git a/doc/user/project/issues/img/related_issues_remove_v12_8.png b/doc/user/project/issues/img/related_issues_remove_v12_8.png Binary files differdeleted file mode 100644 index a8dff4c7052..00000000000 --- a/doc/user/project/issues/img/related_issues_remove_v12_8.png +++ /dev/null diff --git a/doc/user/project/issues/img/related_issues_remove_v15_3.png b/doc/user/project/issues/img/related_issues_remove_v15_3.png Binary files differnew file mode 100644 index 00000000000..577bbed9df3 --- /dev/null +++ b/doc/user/project/issues/img/related_issues_remove_v15_3.png diff --git a/doc/user/project/issues/issue_weight.md b/doc/user/project/issues/issue_weight.md index 756fe9699f1..fcc53a239dc 100644 --- a/doc/user/project/issues/issue_weight.md +++ b/doc/user/project/issues/issue_weight.md @@ -17,6 +17,7 @@ You can set the weight of an issue during its creation, by changing the value in the dropdown menu. You can set it to a non-negative integer value from 0, 1, 2, and so on. You can remove weight from an issue as well. +A user with a Reporter role (or above) can set the weight. This value appears on the right sidebar of an individual issue, as well as in the issues page next to a weight icon (**{weight}**). diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 15d8da7f544..40e5a6d6a92 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -123,8 +123,8 @@ example, if the list is scoped to a label `Frontend`, the new issue also has thi ### By sending an email -> Generated email address format changed in GitLab 11.7. -> The older format is still supported, so existing aliases and contacts still work. +> - Generated email address format changed in GitLab 11.7. +> - The older format is still supported, so existing aliases and contacts still work. You can send an email to create an issue in a project on the project's **Issues List** page. @@ -731,4 +731,4 @@ You can do the following **only by using quick actions**: - [Publish an issue](#publish-an-issue) (`/publish`). - Clone an issue to the same or another project (`/clone`). - Close an issue and mark as a duplicate of another issue (`/duplicate`). -- Copy labels and milestone from another merge request in the project (`/copy_metadata`). +- Copy labels and milestone from another merge request or issue in the project (`/copy_metadata`). diff --git a/doc/user/project/issues/related_issues.md b/doc/user/project/issues/related_issues.md index 028dd2ea473..9dc361b403f 100644 --- a/doc/user/project/issues/related_issues.md +++ b/doc/user/project/issues/related_issues.md @@ -37,7 +37,7 @@ To link one issue to another: - **[is blocked by](#blocking-issues)** 1. Input the issue number or paste in the full URL of the issue. -  +  Issues of the same project can be specified just by the reference number. Issues from a different project require additional information like the @@ -54,7 +54,7 @@ To link one issue to another: When you have finished adding all linked issues, you can see them categorized so their relationships can be better understood visually. - + You can also add a linked issue from a commit message or the description in another issue or MR. [Learn more about crosslinking issues](crosslinking_issues.md). @@ -66,7 +66,7 @@ right-side of each issue token to remove. Due to the bi-directional relationship, the relationship no longer appears in either issue. - + Access our [permissions](../../permissions.md) page for more information. diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index 333b073ee40..62c5489d9c3 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -446,12 +446,13 @@ To learn what happens when you sort by priority or label priority, see ## Real-time changes to labels -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241538) in GitLab 14.10 with a [feature flag](../../administration/feature_flags.md) named `realtime_labels`, disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241538) in GitLab 14.10 with a [feature flag](../../administration/feature_flags.md) named `realtime_labels`, disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/357370#note_991987201) in GitLab 15.1. FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `realtime_labels`. -On GitLab.com, this feature is unavailable. +On GitLab.com, this feature is available. Changed labels are immediately visible to other users, without refreshing the page, on the following: diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index ff5f2ac8cb6..4a6272a0ca3 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -15,7 +15,7 @@ Project members can: 1. Be [direct members](#add-users-to-a-project) of the project. 1. [Inherit membership](#inherited-membership) of the project from the project's group. 1. Be a member of a group that was [shared](share_project_with_groups.md) with the project. -1. Be a member of a group that was [shared with the project's group](../../group/index.md#share-a-group-with-another-group). +1. Be a member of a group that was [shared with the project's group](../../group/manage.md#share-a-group-with-another-group). ```mermaid flowchart RL @@ -49,11 +49,22 @@ flowchart RL [Feature flag `invite_members_group_modal`](https://gitlab.com/gitlab-org/gitlab/-/issues/352526) removed. Add users to a project so they become members and have permission -to perform actions. The Owner [role](../../permissions.md#project-members-permissions) can only be added at the group level. +to perform actions. + +The maximum role (access level) you set depends on if you have the Owner or Maintainer role for the group. For example, the maximum +role that can be set is: + +- Owner (`50`), if you have the Owner role for the project. +- Maintainer (`40`), if you have the Maintainer role on the project. + +In GitLab 14.8 and earlier, direct members of a project have a maximum role of Maintainer. +The Owner [role](../../permissions.md#project-members-permissions) can only be added at the group level. Prerequisite: -- You must have the Maintainer or Owner role. +- You must have the Maintainer or Owner role: + - To remove direct members with the Maintainer role and below, you must have the Maintainer role. + - To remove members with the Owner role, you must have the Owner role. To add a user to a project: @@ -91,7 +102,7 @@ Each user's access is based on: Prerequisite: - You must have the Maintainer or Owner role. -- Sharing the project with other groups must not be [prevented](../../group/index.md#prevent-a-project-from-being-shared-with-groups). +- Sharing the project with other groups must not be [prevented](../../group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups). To add groups to a project: @@ -107,7 +118,7 @@ The members of the group are not displayed on the **Members** tab. The **Members** tab shows: - Members who are directly assigned to the project. -- If the project was created in a group [namespace](../../group/index.md#namespaces), members of that group. +- If the project was created in a group [namespace](../../namespace/index.md), members of that group. ## Import users from another project @@ -173,7 +184,7 @@ To remove a member from a project: user has not forked the private repository or created webhooks. Existing forks continue to receive changes from the upstream project, and webhooks continue to receive updates. You may also want to configure your project to prevent projects in a group - [from being forked outside their group](../../group/index.md#prevent-project-forking-outside-group). + [from being forked outside their group](../../group/access_and_permissions.md#prevent-project-forking-outside-group). 1. Select **Remove member**. ## Filter and sort members diff --git a/doc/user/project/members/share_project_with_groups.md b/doc/user/project/members/share_project_with_groups.md index c4ae00f3c6c..3d5b855a9d3 100644 --- a/doc/user/project/members/share_project_with_groups.md +++ b/doc/user/project/members/share_project_with_groups.md @@ -29,7 +29,7 @@ You can share a project only with: - Groups for which you have an explicitly defined [membership](index.md). - Groups that contain a nested subgroup or project for which you have an explicitly defined role. -Administrators can share projects with any group in the namespace. +Administrators can share projects with any group in the instance. The primary mechanism to give a group of users, say 'Engineering', access to a project, say 'Project Acme', in GitLab is to make the 'Engineering' group the owner of 'Project @@ -52,7 +52,7 @@ After sharing 'Project Acme' with 'Engineering': When you share a project, be aware of the following restrictions and outcomes: - [Maximum access level](#maximum-access-level) -- [Sharing a public project with a private group](#share-a-public-project-with-private-group) +- [Sharing projects with groups of a higher restrictive visibility level](#sharing-projects-with-groups-of-a-higher-restrictive-visibility-level) - [Sharing project with group lock](#share-project-with-group-lock) ## Maximum access level @@ -67,18 +67,24 @@ in. That means you can only share down the hierarchy. For example, `group/subgro - Can not be shared with `group`. - Can be shared with `group/subgroup02` or `group/subgroup01/subgroup03`. -## Share a public project with private group +## Sharing projects with groups of a higher restrictive visibility level -When you share a public project with a private group, be aware of the following outcomes: +There are several outcomes you must be aware of when you share a project with a group that has a more restrictive [visibility level](../../public_access.md#project-and-group-visibility) than the project. For example, when you: -- The name of the group is no longer private and is visible to all users in the project members page. -- Owners of the project have access to members of the private group when they mention them in issues or merge requests. -- Project members who are direct or indirect members of the private group can see private group members listed in addition to members of the project. +- Share a public project with a private group. +- Share a public project with an internal group. +- Share an internal project with a private group. + +The following outcomes occur: + +- The group name is visible to all users that can view the project members page. +- Owners of the project have access to members of the group when they mention them in issues or merge requests. +- Project members who are direct or indirect members of the group can see group members listed in addition to members of the project. ## Share project with group lock -It is possible to prevent projects in a group from [sharing -a project with another group](../members/share_project_with_groups.md). +It is possible to prevent projects in a group from +[sharing a project with another group](../members/share_project_with_groups.md). This allows for tighter control over project access. -Learn more about [Share with group lock](../../group/index.md#prevent-a-project-from-being-shared-with-groups). +Learn more about [Share with group lock](../../group/access_and_permissions.md#prevent-a-project-from-being-shared-with-groups). diff --git a/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png b/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png Binary files differdeleted file mode 100644 index a6636f0bc7f..00000000000 --- a/doc/user/project/merge_requests/approvals/img/scoped_to_protected_branch_v13_10.png +++ /dev/null diff --git a/doc/user/project/merge_requests/approvals/index.md b/doc/user/project/merge_requests/approvals/index.md index 014936208c6..9f33ed6807b 100644 --- a/doc/user/project/merge_requests/approvals/index.md +++ b/doc/user/project/merge_requests/approvals/index.md @@ -23,7 +23,7 @@ flexibility: and require their approval before work can merge. You can configure merge request approvals on a per-project basis, and -[on the group level](../../../group/index.md#group-merge-request-approval-settings). Administrators of +[on the group level](../../../group/manage.md#group-merge-request-approval-settings). Administrators of [GitLab Premium](https://about.gitlab.com/pricing/) and [GitLab Ultimate](https://about.gitlab.com/pricing/) self-managed GitLab instances can also configure approvals diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index b79c8ee867f..c9278c19322 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -7,7 +7,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Merge request approval rules **(PREMIUM)** Approval rules define how many [approvals](index.md) a merge request must receive before it can -be merged, and which users should do the approving. You can define approval rules: +be merged, and which users should do the approving. They can be used in conjunction +with [Code owners](#code-owners-as-eligible-approvers) to ensure that changes are +reviewed both by the group maintaining the feature, and any groups responsible +for specific areas of oversight. + +You can define approval rules: - [As project defaults](#add-an-approval-rule). - [Per merge request](#edit-or-override-merge-request-approval-rules). @@ -127,7 +132,7 @@ users were not explicitly listed in the approval rules. ### Group approvers You can add a group of users as approvers, but those users count as approvers only if -they have direct membership to the group. Group approvers are +they have **direct membership** to the group. Inherited members do not count. Group approvers are restricted to only groups [with share access to the project](../../members/share_project_with_groups.md). A user's membership in an approvers group affects their individual ability to @@ -172,15 +177,15 @@ oversight on proposed work. To enable approval permissions for these users witho granting them push access: 1. [Create a protected branch](../../protected_branches.md) -1. [Create a new group](../../../group/index.md#create-a-group). -1. [Add the user to the group](../../../group/index.md#add-users-to-a-group), +1. [Create a new group](../../../group/manage.md#create-a-group). +1. [Add the user to the group](../../../group/manage.md#add-users-to-a-group), and select the Reporter role for the user. 1. [Share the project with your group](../../members/share_project_with_groups.md#share-a-project-with-a-group-of-users), based on the Reporter role. 1. Go to your project and select **Settings > General**. 1. Expand **Merge request (MR) approvals**. 1. Select **Add approval rule** or **Update approval rule** and target the protected branch. -1. [Add the group](../../../group/index.md#create-a-group) to the permission list. +1. [Add the group](../../../group/manage.md#create-a-group) to the permission list.  @@ -213,7 +218,8 @@ appreciated, but not required. To make an approval rule optional: ## Approvals for protected branches -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.8. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/460) in GitLab 12.8. +> - **All protected branches** target branch option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360930) in GitLab 15.3. Approval rules are often relevant only to specific branches, like your [default branch](../../repository/branches/default.md). To configure an @@ -223,10 +229,10 @@ approval rule for certain branches: 1. Go to your project and select **Settings**. 1. Expand **Merge request (MR) approvals**. 1. Select a **Target branch**: - - To protect all branches, select **All branches**. - - To select a specific branch, select it from the list: + - To apply the rule to all branches, select **All branches**. + - To apply the rule to all protected branches, select **All protected branches** (GitLab 15.3 and later). + - To apply the rule to a specific branch, select it from the list: -  1. To enable this configuration, read [Code Owner's approvals for protected branches](../../protected_branches.md#require-code-owner-approval-on-a-protected-branch). diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index 7b865a91106..3ca8ddb508a 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -19,7 +19,9 @@ To view or edit merge request approval settings: 1. Go to your project and select **Settings > General**. 1. Expand **Merge request (MR) approvals**. -In this section of general settings, you can configure the following settings: +### Approval settings + +These settings limit who can approve merge requests. | Setting | Description | | ------ | ------ | @@ -27,7 +29,14 @@ In this section of general settings, you can configure the following settings: | [Prevent approvals by users who add commits](#prevent-approvals-by-users-who-add-commits) | When enabled, users who have committed to a merge request cannot approve it. | | [Prevent editing approval rules in merge requests](#prevent-editing-approval-rules-in-merge-requests) | When enabled, users can't override the project's approval rules on merge requests. | | [Require user password to approve](#require-user-password-to-approve) | Force potential approvers to first authenticate with a password. | -| [Remove all approvals when commits are added to the source branch](#remove-all-approvals-when-commits-are-added-to-the-source-branch) | When enabled, remove all existing approvals on a merge request when more changes are added to it. | + +You can further define what happens to existing approvals when commits are added to the merge request. + +| Setting | Description | +| ------ | ------ | +| Keep approvals | Do not remove approvals. | +| [Remove all approvals](#remove-all-approvals-when-commits-are-added-to-the-source-branch) | Remove all existing approvals. | +| [Remove approvals by Code Owners if their files changed](#remove-approvals-by-code-owners-if-their-files-changed) | If a Code Owner has approved the merge request, and the commit changes files they are the Code Owner for, their approval is removed. | ## Prevent approval by author @@ -119,9 +128,27 @@ when more changes are added to it: 1. Select the **Remove all approvals when commits are added to the source branch** checkbox. 1. Select **Save changes**. -Approvals aren't reset when a merge request is [rebased from the UI](../methods/index.md#rebasing-in-semi-linear-merge-methods). +Approvals aren't removed when a merge request is [rebased from the UI](../methods/index.md#rebasing-in-semi-linear-merge-methods) However, approvals are reset if the target branch is changed. +## Remove approvals by Code Owners if their files changed + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90578) in GitLab 15.3. + +If you only want to remove approvals by Code Owners whose files have been changed: + +Prerequisite: + +- You must have at least the Maintainer role for a project. + +To do this: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Merge request approvals**. +1. Select **Remove approvals by Code Owners if their files changed**. +1. Select **Save changes**. + ## Code coverage check approvals You can require specific approvals if a merge request would result in a decline in code test @@ -139,7 +166,7 @@ You can also enforce merge request approval settings: - At the [instance level](../../../admin_area/merge_requests_approvals.md), which apply to all groups on an instance and, therefore, all projects. -- On a [top-level group](../../../group/index.md#group-merge-request-approval-settings), which apply to all subgroups +- On a [top-level group](../../../group/manage.md#group-merge-request-approval-settings), which apply to all subgroups and projects. If the settings are inherited by a group or project, they cannot be changed in the group or project diff --git a/doc/user/project/merge_requests/csv_export.md b/doc/user/project/merge_requests/csv_export.md index 2adcc4d4575..893b2bc6811 100644 --- a/doc/user/project/merge_requests/csv_export.md +++ b/doc/user/project/merge_requests/csv_export.md @@ -8,13 +8,21 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3619) in GitLab 13.6. -Exporting merge requests CSV enables you and your team to export all the data collected from merge requests into a comma-separated values (CSV) file, which stores tabular data in plain text. +Export all the data collected from a project's merge requests into a comma-separated values (CSV) file. -To export merge requests to CSV, navigate to your **Merge requests** from the sidebar of a project and select **Export as CSV**. +To export merge requests to a CSV file: + +1. On the top bar, select **Menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** . +1. Add any searches or filters. This can help you keep the size of the CSV file under the 15MB limit. The limit ensures + the file can be emailed to a variety of email providers. +1. Select **Export as CSV** (**{export}**). +1. Confirm the correct number of merge requests are to be exported. +1. Select **Export merge requests**. ## CSV Output -The following table shows what attributes will be present in the CSV. +The following table shows the attributes in the CSV file. | Column | Description | |--------------------|--------------------------------------------------------------| @@ -42,8 +50,3 @@ The following table shows what attributes will be present in the CSV. In GitLab 14.7 and earlier, the first two columns were `MR ID` and `URL`, which [caused an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/34769) when importing back into GitLab. - -## Limitations - -- Export merge requests to CSV is not available at the Group's merge request list. -- As the merge request CSV file is sent as an email attachment, the size is limited to 15MB to ensure successful delivery across a range of email providers. If you need to minimize the size of the file, you can narrow the search before export. For example, you can set up exports of open and closed merge requests in separate files. diff --git a/doc/user/project/merge_requests/img/license_compliance_widget_v15_3.png b/doc/user/project/merge_requests/img/license_compliance_widget_v15_3.png Binary files differnew file mode 100644 index 00000000000..a9f73d40f82 --- /dev/null +++ b/doc/user/project/merge_requests/img/license_compliance_widget_v15_3.png diff --git a/doc/user/project/merge_requests/img/status_checks_widget_passed_v14_0.png b/doc/user/project/merge_requests/img/status_checks_widget_passed_v14_0.png Binary files differdeleted file mode 100644 index de61ca8b553..00000000000 --- a/doc/user/project/merge_requests/img/status_checks_widget_passed_v14_0.png +++ /dev/null diff --git a/doc/user/project/merge_requests/img/status_checks_widget_pending_v14_0.png b/doc/user/project/merge_requests/img/status_checks_widget_pending_v14_0.png Binary files differdeleted file mode 100644 index c4e606bd2f4..00000000000 --- a/doc/user/project/merge_requests/img/status_checks_widget_pending_v14_0.png +++ /dev/null diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index a7a669d3b75..35ec075c674 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -41,7 +41,7 @@ To view merge requests for all projects in a group: If your group contains subgroups, this view also displays merge requests from the subgroup projects. -## View all merge requests assigned to you +### View all merge requests assigned to you To view all merge requests assigned to you: @@ -52,13 +52,14 @@ To view all merge requests assigned to you: <!-- vale gitlab.FirstPerson = YES --> -Or: +or: - To use a [keyboard shortcut](../../shortcuts.md), press <kbd>Shift</kbd> + <kbd>m</kbd>. -- On the top bar, on the top right, select **{merge-request-open}** **Merge requests**. - Then select one of the following: - - [Review requests](reviews/index.md). - - Merge requests assigned. + +or: + +1. On the top bar, on the top right, select **{merge-request-open}** **Merge requests**. +1. From the dropdown list, select **Assigned to you**. ## Filter the list of merge requests diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md index 63b464e5ff4..c4e4b40dc48 100644 --- a/doc/user/project/merge_requests/methods/index.md +++ b/doc/user/project/merge_requests/methods/index.md @@ -150,12 +150,8 @@ considered equivalent to rebasing. ### Rebase without CI/CD pipeline -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) in GitLab 14.7 [with a flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. Disabled by default. - -FLAG: -On GitLab.com and self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. -The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118825) in GitLab 14.7 [with a flag](../../../../administration/feature_flags.md) named `rebase_without_ci_ui`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/350262) in GitLab 15.3. Feature flag `rebase_without_ci_ui` removed. To rebase a merge request's branch without triggering a CI/CD pipeline, select **Rebase without pipeline** from the merge request reports section. diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v13_11.png b/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v13_11.png Binary files differdeleted file mode 100644 index 6b4899bf67f..00000000000 --- a/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v13_11.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v15_3.png b/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v15_3.png Binary files differnew file mode 100644 index 00000000000..b73dbb50cd2 --- /dev/null +++ b/doc/user/project/merge_requests/reviews/img/mr_review_new_comment_v15_3.png diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_resolve.png b/doc/user/project/merge_requests/reviews/img/mr_review_resolve.png Binary files differdeleted file mode 100644 index ced33682459..00000000000 --- a/doc/user/project/merge_requests/reviews/img/mr_review_resolve.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_resolve2.png b/doc/user/project/merge_requests/reviews/img/mr_review_resolve2.png Binary files differdeleted file mode 100644 index 2f0be3b6d06..00000000000 --- a/doc/user/project/merge_requests/reviews/img/mr_review_resolve2.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_start.png b/doc/user/project/merge_requests/reviews/img/mr_review_start.png Binary files differdeleted file mode 100644 index 08b4c6bb82b..00000000000 --- a/doc/user/project/merge_requests/reviews/img/mr_review_start.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/img/mr_review_unresolve.png b/doc/user/project/merge_requests/reviews/img/mr_review_unresolve.png Binary files differdeleted file mode 100644 index 4bef38f7808..00000000000 --- a/doc/user/project/merge_requests/reviews/img/mr_review_unresolve.png +++ /dev/null diff --git a/doc/user/project/merge_requests/reviews/img/mr_summary_comment_v15_3.png b/doc/user/project/merge_requests/reviews/img/mr_summary_comment_v15_3.png Binary files differnew file mode 100644 index 00000000000..38e18115803 --- /dev/null +++ b/doc/user/project/merge_requests/reviews/img/mr_summary_comment_v15_3.png diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index a8f43dd9c02..27b223c48ec 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -36,13 +36,9 @@ To start your review: 1. Select the **{comment}** **comment** icon in the gutter to expand the diff lines and display a comment box. In GitLab version 13.2 and later, you can [select multiple lines](#comment-on-multiple-lines). -1. Write your first comment, and select **Start a review** below your comment: -  -1. Continue adding comments to lines of code, and select the appropriate button after - you write a comment: - - **Add to review**: Keep this comment private and add to the current review. - These review comments are marked **Pending** and are visible only to you. - - **Add comment now**: Submits the specific comment as a regular comment instead of as part of the review. +1. In the text area, write your first comment, then select **Start a review** below your comment. +1. Continue adding comments to lines of code. After each comment, select **Add to review**. + Comments made as part of a review are visible only to you until you submit your review. 1. Optional. You can use [quick actions](../../quick_actions.md) inside review comments. The comment shows the actions to perform after publication, but does not perform them until you submit your review. @@ -60,8 +56,12 @@ displays next to your name. You can submit your completed review in multiple ways: - Use the `/submit_review` [quick action](../../quick_actions.md) in the text of a non-review comment. -- When creating a review comment, select **Submit review**. -- Scroll to the bottom of the screen and select **Submit review**. +- Select **Finish review** and then **Submit review** in the footer at the bottom of the screen. + +Selecting **Finish review** opens a modal window to add an optional comment to summarize your review. +You can also include quick actions: + + When you submit your review, GitLab: @@ -73,25 +73,25 @@ When you submit your review, GitLab: ### Resolve or unresolve thread with a comment Review comments can also resolve or unresolve [resolvable threads](../../../discussions/index.md#resolve-a-thread). -When replying to a comment, a checkbox is displayed to resolve or unresolve -the thread after publication. - - +To resolve or unresolve a thread when replying to a comment: -If a particular pending comment resolves or unresolves the thread, this is shown on the pending -comment itself. +1. In the comment text area, write your comment. +1. Select or clear **Resolve thread**. +1. Select **Add comment now** or **Add to review**. - +Pending comments display information about the action to be taken when the comment is published: - +- **{check-circle-filled}** Thread will be resolved. +- **{check-circle}** Thread stays unresolved. ### Add a new comment > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8225) in GitLab 13.10. -If you have a review in progress, you are presented with the option to **Add to review**: +If you have a review in progress, you can also add a comment from the **Overview** tab by selecting + **Add to review**: - + ### Approval Rule information for Reviewers **(PREMIUM)** diff --git a/doc/user/project/merge_requests/status_checks.md b/doc/user/project/merge_requests/status_checks.md index 423179325d3..0d7794a3ebd 100644 --- a/doc/user/project/merge_requests/status_checks.md +++ b/doc/user/project/merge_requests/status_checks.md @@ -10,6 +10,7 @@ disqus_identifier: 'https://docs.gitlab.com/ee/user/project/merge_requests/statu > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3869) in GitLab 14.0, disabled behind the `:ff_external_status_checks` feature flag. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/320783) in GitLab 14.1. +> - `failed` status [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329636) in GitLab 14.9. You can create a status check that sends merge request data to third-party tools. When users create, change, or close merge requests, GitLab sends a notification. The users or automated workflows @@ -50,9 +51,8 @@ Merge requests return a `409 Conflict` error to any responses that do not refer External status checks have the following states: - `pending` - The default state. No response can been received by the merge request from the external service. -- `response received` - A response from the external service has been received and approved by it. - -Support for adding a `failed` state is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338827). +- `passed` - A response from the external service has been received and approved by it. +- `failed` - A response from the external service has been received and denied by it. If something changes outside of GitLab, you can [set the status of an external status check](../../../api/status_checks.md#set-status-of-an-external-status-check) using the API. You don't need to wait for a merge request webhook payload to be sent first. @@ -138,24 +138,18 @@ the status check and it **will not** be recoverable. ## Status checks widget -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327634) in GitLab 14.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327634) in GitLab 14.1. +> - UI [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91504) in GitLab 15.2. -The status checks widget displays in merge requests and shows the status of external -status checks: +The status checks widget displays in merge requests and displays the following statuses: - +- **pending** (**{status-neutral}**), while GitLab waits for a response from an external status check. +- **success** (**{status-success}**) or **failed** (**{status-failed}**), when GitLab receives a response from an external status check. An organization might have a policy that does not allow merging merge requests if external status checks do not pass. However, the details in the widget are for informational purposes only. GitLab does not prevent merging of merge requests that fail status checks. - -While GitLab waits for a response from the external status check, the widget shows -the status checks as `pending`: - - - -After GitLab [receives a response](../../../api/status_checks.md#set-status-of-an-external-status-check) -from the external status check, the widget updates accordingly. +Support to allow merges to be blocked when external status checks fail is proposed in epic [&8516](https://gitlab.com/groups/gitlab-org/-/epics/8516). NOTE: GitLab cannot guarantee that the external status checks are properly processed by diff --git a/doc/user/project/merge_requests/widgets.md b/doc/user/project/merge_requests/widgets.md index b0464f3f972..0e179415192 100644 --- a/doc/user/project/merge_requests/widgets.md +++ b/doc/user/project/merge_requests/widgets.md @@ -63,6 +63,12 @@ faster to preview proposed modifications. [Read more about Review Apps](../../../ci/review_apps/index.md). +## License compliance **(ULTIMATE)** + +If you have configured [License Compliance](../../compliance/license_compliance/index.md) for your project, then you can view a list of licenses that are detected for your project's dependencies. + + + ## External status checks **(ULTIMATE)** If you have configured [external status checks](status_checks.md) you can diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index ba48876d4fd..ef734225fb4 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -44,6 +44,14 @@ To view the milestone list: In a project, GitLab displays milestones that belong to the project. In a group, GitLab displays milestones that belong to the group and all projects in the group. +NOTE: +If a project has issue tracking +[turned off](../settings/index.md#configure-project-visibility-features-and-permissions), +you can get to the milestones page +by going to its URL. To do so, add: `/-/milestones` to your project or group URL. +For example `https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project/-/milestones`. +This is tracked in [issue 339009](https://gitlab.com/gitlab-org/gitlab/-/issues/339009). + ### View all milestones You can view all the milestones you have access to in the entire GitLab namespace. diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index 6daf671a751..c4c30dbdab4 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -32,6 +32,7 @@ for the most popular hosting services: <!-- vale gitlab.Spelling = NO --> +- [123-reg](https://www.123-reg.co.uk/support/domains/domain-name-server-dns-management-guide/) - [Amazon](https://docs.aws.amazon.com/AmazonS3/latest/dev/website-hosting-custom-domain-walkthrough.html) - [Bluehost](https://www.bluehost.com/help/article/dns-management-add-edit-or-delete-dns-entries) - [Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website) diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index 184e4f345c1..0cc6cb808d1 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -79,8 +79,6 @@ If you get an error **Something went wrong while obtaining the Let's Encrypt cer 1. Make sure [your domain is verified](index.md#1-add-a-custom-domain-to-pages). 1. Go to step 1. -Another possible cause of this error is the `_redirects` file because the current implementation relies on an HTTP ACME challenge. If you redirect the `.acme-challenge/` endpoint Let's Encrypt cannot validate the domain. Make sure you don't have a wildcard (`*`) redirect either as that too breaks validation. The problem with wildcard redirects is tracked in the [Wildcard redirects break Let's Encrypt integration](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/649) issue. - ### Message "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later." hangs for more than an hour If you've enabled Let's Encrypt integration, but a certificate is absent after an hour and you see the message, "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later.", try to remove and add the domain for GitLab Pages again by following these steps: diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md index 5fd17b5c07e..68a2a6a80ad 100644 --- a/doc/user/project/pages/getting_started/pages_from_scratch.md +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -420,10 +420,10 @@ Now GitLab CI/CD not only builds the website, but also: For more information, see the following blog posts. -- [Use GitLab CI/CD `environments` to deploy your - web app to staging and production](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/). -- Learn [how to run jobs sequentially, - in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/). +- Use GitLab CI/CD `environments` to + [deploy your web app to staging and production](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/). +- Learn how to run jobs + [sequentially, in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2016/07/29/the-basics-of-gitlab-ci/). - Learn [how to pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) to deploy this website, <https://docs.gitlab.com>. - Learn [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index 54b843945cd..29712a82be1 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -18,7 +18,7 @@ replace the Pages wildcard domain on GitLab.com (`*.gitlab.io`) with your own. If you set up a GitLab Pages project on GitLab, it's automatically accessible under a subdomain of `namespace.example.io`. -The [`namespace`](../../group/index.md#namespaces) +The [`namespace`](../../namespace/index.md) is defined by your username on GitLab.com, or the group name you created this project under. For GitLab self-managed instances, replace `example.io` diff --git a/doc/user/project/pages/img/remove_pages.png b/doc/user/project/pages/img/remove_pages.png Binary files differdeleted file mode 100644 index d6c37ef30cd..00000000000 --- a/doc/user/project/pages/img/remove_pages.png +++ /dev/null diff --git a/doc/user/project/pages/img/remove_pages_v15_3.png b/doc/user/project/pages/img/remove_pages_v15_3.png Binary files differnew file mode 100644 index 00000000000..f740daf5c0b --- /dev/null +++ b/doc/user/project/pages/img/remove_pages_v15_3.png diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 1ea7500273e..3bd16a17f23 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -74,7 +74,7 @@ to your project's settings through the gear icon in the top right, and then navigating to **Pages**. Select the **Remove pages** button to delete your Pages website. - + ## Subdomains of subdomains @@ -297,9 +297,6 @@ A 404 can also be related to incorrect permissions. If [Pages Access Control](pa navigates to the Pages URL and receives a 404 response, it is possible that the user does not have permission to view the site. To fix this, verify that the user is a member of the project. -For Geo instances, 404 errors on Pages occur after promoting a secondary to a primary. -Find more details in the [Pages administration documentation](../../../administration/pages/index.md#404-error-after-promoting-a-geo-secondary-to-a-primary-node) - ### Cannot play media content on Safari Safari requires the web server to support the [Range request header](https://developer.apple.com/library/archive/documentation/AppleApplications/Reference/SafariWebContent/CreatingVideoforSafarioniPhone/CreatingVideoforSafarioniPhone.html#//apple_ref/doc/uid/TP40006514-SW6) diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index eb897c176fa..e6446c34bf0 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -6,8 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitLab Pages access control **(FREE)** -> Available on GitLab.com in GitLab 12.4. - You can enable Pages access control on your project if your administrator has [enabled the access control feature](../../../administration/pages/index.md#access-control) on your GitLab instance. When enabled, only authenticated diff --git a/doc/user/project/push_options.md b/doc/user/project/push_options.md index 6ef8477b6b6..d02609cbdc7 100644 --- a/doc/user/project/push_options.md +++ b/doc/user/project/push_options.md @@ -95,15 +95,15 @@ git push -o merge_request.create -o merge_request.target=my-target-branch -o mer ## Useful Git aliases As shown above, Git push options can cause Git commands to grow very long. If -you use the same push options frequently, it's useful to create [Git -aliases](https://git-scm.com/book/en/v2/Git-Basics-Git-Aliases). Git aliases +you use the same push options frequently, it's useful to create +[Git aliases](https://git-scm.com/book/en/v2/Git-Basics-Git-Aliases). Git aliases are command line shortcuts for Git which can significantly simplify the use of long Git commands. ### Merge when pipeline succeeds alias -To set up a Git alias for the [merge when pipeline succeeds Git push -option](#push-options-for-merge-requests): +To set up a Git alias for the +[merge when pipeline succeeds Git push option](#push-options-for-merge-requests): ```shell git config --global alias.mwps "push -o merge_request.create -o merge_request.target=master -o merge_request.merge_when_pipeline_succeeds" diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 96e51b061ee..216d040734d 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -56,6 +56,7 @@ threads. Some quick actions might not be available to all subscription tiers. | `/assign_reviewer @user1 @user2` or `/reviewer @user1 @user2` or `/request_review @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users as reviewers. | | `/assign_reviewer me` or `/reviewer me` or `/request_review me` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself as a reviewer. | | `/award :emoji:` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Toggle emoji award. | +| `/cc @user` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mention a user. In GitLab 15.0 and later, this command performs no action. You can instead type `CC @user` or only `@user`. [In GitLab 14.9 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/31200), mentioning a user at the start of a line created a specific type of to-do item notification. | | `/child_epic <epic>` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Add child epic to `<epic>`. The `<epic>` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7330) in GitLab 12.0). | | `/clear_health_status` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear [health status](issues/managing_issues.md#health-status) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). | | `/clear_weight` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear weight. | diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index 1d448ca5c94..d3456e086ce 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -32,12 +32,10 @@ When you create a release, or after, you can: - Add release notes. - Add a message for the Git tag associated with the release. - [Associate milestones with it](#associate-milestones-with-a-release). -- Attach [release assets](#release-assets), like runbooks or packages. +- Attach [release assets](release_fields.md#release-assets), like runbooks or packages. ## View releases -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36667) in GitLab 12.8. - To view a list of releases: - On the left sidebar, select **Deployments > Releases**, or @@ -81,18 +79,18 @@ To create a release in the Releases page: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Deployments > Releases** and select **New release**. -1. From the [**Tag name**](#tag-name) dropdown, either: +1. From the [**Tag name**](release_fields.md#tag-name) dropdown, either: - Select an existing Git tag. Selecting an existing tag that is already associated with a release results in a validation error. - Enter a new Git tag name. 1. From the **Create from** dropdown, select a branch or commit SHA to use when creating the new tag. 1. Optional. Enter additional information about the release, including: - - [Title](#title). + - [Title](release_fields.md#title). - [Milestones](#associate-milestones-with-a-release). - - [Release notes](#release-notes-description). + - [Release notes](release_fields.md#release-notes-description). - Whether or not to include the [Tag message](../../../topics/git/tags.md). - - [Asset links](#links). + - [Asset links](release_fields.md#links). 1. Select **Create release**. ### Create a release in the Tags page @@ -126,99 +124,8 @@ You can create a release directly as part of the GitLab CI/CD pipeline by using The release is created only if the job processes without error. If the API returns an error during release creation, the release job fails. -Methods for creating a release using a CI/CD job include: - -- Create a release when a Git tag is created. -- Create a release when a commit is merged to the default branch. - -#### Create a release when a Git tag is created - -In this CI/CD example, pushing a Git tag to the repository, or creating a Git tag in the UI triggers -the release. You can use this method if you prefer to create the Git tag manually, and create a -release as a result. - -NOTE: -Do not provide Release notes when you create the Git tag in the UI. Providing release notes -creates a release, resulting in the pipeline failing. - -Key points in the following _extract_ of an example `.gitlab-ci.yml` file: - -- The `rules` stanza defines when the job is added to the pipeline. -- The Git tag is used in the release's name and description. - -```yaml -release_job: - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - rules: - - if: $CI_COMMIT_TAG # Run this job when a tag is created - script: - - echo "running release_job" - release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - tag_name: '$CI_COMMIT_TAG' - description: '$CI_COMMIT_TAG' -``` - -#### Create a release when a commit is merged to the default branch - -In this CI/CD example, merging a commit to the default branch triggers the pipeline. You can use -this method if your release workflow does not create a tag manually. - -Key points in the following _extract_ of an example `.gitlab-ci.yml` file: - -- The Git tag, description, and reference are created automatically in the pipeline. -- If you manually create a tag, the `release_job` job does not run. - -```yaml -release_job: - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - rules: - - if: $CI_COMMIT_TAG - when: never # Do not run this job when a tag is created manually - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch - script: - - echo "running release_job for $TAG" - release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - tag_name: 'v0.$CI_PIPELINE_IID' # The version is incremented per pipeline. - description: 'v0.$CI_PIPELINE_IID' - ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. -``` - -NOTE: -Environment variables set in `before_script` or `script` are not available for expanding -in the same job. Read more about -[potentially making variables available for expanding](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6400). - -#### Skip multiple pipelines when creating a release - -Creating a release using a CI/CD job could potentially trigger multiple pipelines if the associated tag does not exist already. To understand how this might happen, consider the following workflows: - -- Tag first, release second: - 1. A tag is created via UI or pushed. - 1. A tag pipeline is triggered, and runs `release` job. - 1. A release is created. - -- Release first, tag second: - 1. A pipeline is triggered when commits are pushed or merged to default branch. The pipeline runs `release` job. - 1. A release is created. - 1. A tag is created. - 1. A tag pipeline is triggered. The pipeline also runs `release` job. - -In the second workflow, the `release` job runs in multiple pipelines. To prevent this, you can use the [`workflow:rules` keyword](../../../ci/yaml/index.md#workflowrules) to determine if a release job should run in a tag pipeline: - -```yaml -release_job: - rules: - - if: $CI_COMMIT_TAG - when: never # Do not run this job in a tag pipeline - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch - script: - - echo "Create release" - release: - name: 'My awesome release' - tag_name: '$CI_COMMIT_TAG' -``` +For examples of how you can create a release of your application in the CI/CD pipeline, +see [Release CI/CD examples](release_cicd_examples.md). ### Use a custom SSL CA certificate authority @@ -250,18 +157,6 @@ The `ADDITIONAL_CA_CERT_BUNDLE` value can also be configured as a either as a `file`, which requires the path to the certificate, or as a variable, which requires the text representation of the certificate. -### `release-cli` command line - -The entries under the `release` node are transformed into Bash commands and sent -to the Docker container, which contains the [release-cli](https://gitlab.com/gitlab-org/release-cli). -You can also call the `release-cli` directly from a `script` entry. - -For example, if you use the YAML described previously: - -```shell -release-cli create --name "Release $CI_COMMIT_SHA" --description "Created using the release-cli $EXTRA_DESCRIPTION" --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" --assets-link "{\"name\":\"asset1\",\"url\":\"https://example.com/assets/1\",\"link_type\":\"other\"} -``` - ### Create multiple releases in a single pipeline A pipeline can have multiple `release` jobs, for example: @@ -298,10 +193,17 @@ release tag. When the `released_at` date and time has passed, the badge is autom  -## Edit a release +## Historical releases + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/199429) in GitLab 15.2. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26016) in GitLab 12.6. -> - Asset link editing [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9427) in GitLab 12.10. +You can create a release in the past using either the +[Releases API](../../../api/releases/index.md#historical-releases) or the UI. When you set +a past `released_at` date, an **Historical release** badge is displayed next to +the release tag. Due to being released in the past, [release evidence](#release-evidence) +is not available. + +## Edit a release Only users with at least the Developer role can edit releases. Read more about [Release permissions](#release-permissions). @@ -430,277 +332,6 @@ complete overlapping period. For more information, see [Deployment safety](../../../ci/environments/deployment_safety.md). -## Release fields - -The following fields are available when you create or edit a release. - -### Title - -The release title can be customized using the **Release title** field when -creating or editing a release. If no title is provided, the release's tag name -is used instead. - -### Tag name - -The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) -for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the -[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). - -For example, for GitLab version `10.5.7`: - -- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. -- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. -- `7` represents the patch number. - -Any part of the version number can be multiple digits, for example, `13.10.11`. - -### Release notes description - -Every release has a description. You can add any text you like, but we recommend -including a changelog to describe the content of your release. This helps users -quickly scan the differences between each release you publish. - -[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can -be included in Release note descriptions by selecting **Include tag message in -the release notes**. - -Description supports [Markdown](../../markdown.md). - -### Release assets - -A release contains the following types of assets: - -- [Source code](#source-code) -- [Link](#links) - -#### Source code - -GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar` -archived source code from the given Git tag. These are read-only assets. - -#### Links - -A link is any URL which can point to whatever you like: documentation, built -binaries, or other related materials. These can be both internal or external -links from your GitLab instance. -Each link as an asset has the following attributes: - -| Attribute | Description | Required | -| ---- | ----------- | --- | -| `name` | The name of the link. | Yes | -| `url` | The URL to download a file. | Yes | -| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No | -| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No | - -##### Permanent link to latest release - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. - -Latest release page is accessible through a permanent URL. -GitLab will redirect to the latest release page URL when it is visited. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/permalink/latest -``` - -We also support, suffix path carry forward on the redirect to the latest release. -Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`. - -Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage. - -###### Sorting preferences - -By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945). - -##### Permanent links to release assets - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. - -The assets associated with a release are accessible through a permanent URL. -GitLab always redirects this URL to the actual asset -location, so even if the assets move to a different location, you can continue -to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/:release/downloads/:filepath -``` - -If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` -namespace and `gitlab-runner` project on `gitlab.com`, for example: - -```json -{ - "name": "linux amd64", - "filepath": "/binaries/gitlab-runner-linux-amd64", - "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", - "link_type": "other" -} -``` - -This asset has a direct link of: - -```plaintext -https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 -``` - -The physical location of the asset can change at any time and the direct link remains unchanged. - -##### Permanent links to latest release assets - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. - -The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*. - -The format of the URL is: - -```plaintext -https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath -``` - -If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org` -namespace and `gitlab-runner` project on `gitlab.com`, for example: - -```json -{ - "name": "linux amd64", - "filepath": "/binaries/gitlab-runner-linux-amd64", - "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", - "link_type": "other" -} -``` - -This asset has a direct link of: - -```plaintext -https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64 -``` - -##### Link Types - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1. - -The four types of links are "Runbook," "Package," "Image," and "Other." -The `link_type` parameter accepts one of the following four values: - -- `runbook` -- `package` -- `image` -- `other` (default) - -This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. - -##### Use a generic package for attaching binaries - -You can use [generic packages](../../packages/generic_packages/index.md) -to store any artifacts from a release or tag pipeline, -that can also be used for attaching binary files to an individual release entry. -You basically need to: - -1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file). -1. [Attach the package link to the release](#links). - -The following example generates release assets, publishes them -as a generic package, and then creates a release: - -```yaml -stages: - - build - - upload - - release - -variables: - # Package version can only contain numbers (0-9), and dots (.). - # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion. - # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file - PACKAGE_VERSION: "1.2.3" - DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}" - LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}" - PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}" - -build: - stage: build - image: alpine:latest - rules: - - if: $CI_COMMIT_TAG - script: - - mkdir bin - - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY} - - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY} - artifacts: - paths: - - bin/ - -upload: - stage: upload - image: curlimages/curl:latest - rules: - - if: $CI_COMMIT_TAG - script: - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}" - - | - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}" - -release: - # Caution, as of 2021-02-02 these assets links require a login, see: - # https://gitlab.com/gitlab-org/gitlab/-/issues/299384 - stage: release - image: registry.gitlab.com/gitlab-org/release-cli:latest - rules: - - if: $CI_COMMIT_TAG - script: - - | - release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ - --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \ - --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}" -``` - -PowerShell users may need to escape the double quote `"` inside a JSON -string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json` -before passing on to the `release-cli`. -For example: - -```yaml -release: - script: - - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}" - - $env:assetjson = $env:asset | ConvertTo-Json - - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson -``` - -NOTE: -Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md) -links to a release is not recommended, because artifacts are ephemeral and -are used to pass data in the same pipeline. This means there's a risk that -they could either expire or someone might manually delete them. - -#### Number of new and total features **(FREE SAAS)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5. - -On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project. - - - -The totals are displayed on [shields](https://shields.io/) and are generated per release by -[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake). - -| Item | Formula | -| ------ | ------ | -| `New features` | Total count of release posts across all tiers for a single release in the project. | -| `Total features` | Total count of release posts in reverse order for all releases in the project. | - -The counts are also shown by license tier. - -| Item | Formula | -| ------ | ------ | -| `New features` | Total count of release posts across a single tier for a single release in the project. | -| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. | - ## Release evidence > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26019) in GitLab 12.6. @@ -828,10 +459,11 @@ keyword. Learn more in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issue In the API: -- If you specify a future `released_at` date, the release becomes an **Upcoming Release** +- If you specify a future `released_at` date, the release becomes an **Upcoming release** and the evidence is collected on the date of the release. You cannot collect release evidence before then. -- If you use a past `released_at` date, no evidence is collected. +- If you specify a past `released_at` date, the release becomes an **Historical + release** and no evidence is collected. - If you do not specify a `released_at` date, release evidence is collected on the date the release is created. @@ -848,7 +480,7 @@ In the API: - Users with the Guest role have read and download access to the project releases. This includes associated Git-tag-names, release description, author information of the releases. - However, other repository-related information, such as [source code](#source-code), [release evidence](#release-evidence) are redacted. + However, other repository-related information, such as [source code](release_fields.md#source-code), [release evidence](#release-evidence) are redacted. ### Create, update, and delete a release and its assets @@ -862,19 +494,6 @@ users with at least the Maintainer role to create, update, and delete releases by protecting the tag with a wildcard (`*`), and set **Maintainer** in the **Allowed to create** column. -## Release Command Line - -> [Introduced](https://gitlab.com/gitlab-org/release-cli/-/merge_requests/6) in GitLab 12.10. - -The Release CLI is a command-line tool for managing GitLab Releases from the command line or from -the GitLab CI/CD configuration file, `.gitlab-ci.yml`. - -With it, you can create, update, modify, and delete releases right through the -terminal. - -Read the [Release CLI documentation](https://gitlab.com/gitlab-org/release-cli/-/blob/master/docs/index.md) -for details. - ## Release Metrics **(ULTIMATE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259703) in GitLab Premium 13.9. diff --git a/doc/user/project/releases/release_cicd_examples.md b/doc/user/project/releases/release_cicd_examples.md new file mode 100644 index 00000000000..f1d3e55a707 --- /dev/null +++ b/doc/user/project/releases/release_cicd_examples.md @@ -0,0 +1,100 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Release CI/CD examples + +GitLab release functionality is flexible, able to be configured to match your workflow. This page +features example CI/CD release jobs. Each example demonstrates a method of creating a release in a +CI/CD pipeline. + +## Create a release when a Git tag is created + +In this CI/CD example, pushing a Git tag to the repository, or creating a Git tag in the UI triggers +the release. You can use this method if you prefer to create the Git tag manually, and create a +release as a result. + +NOTE: +Do not provide Release notes when you create the Git tag in the UI. Providing release notes +creates a release, resulting in the pipeline failing. + +Key points in the following _extract_ of an example `.gitlab-ci.yml` file: + +- The `rules` stanza defines when the job is added to the pipeline. +- The Git tag is used in the release's name and description. + +```yaml +release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG # Run this job when a tag is created + script: + - echo "running release_job" + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: '$CI_COMMIT_TAG' + description: '$CI_COMMIT_TAG' +``` + +## Create a release when a commit is merged to the default branch + +In this CI/CD example, merging a commit to the default branch triggers the pipeline. You can use +this method if your release workflow does not create a tag manually. + +Key points in the following _extract_ of an example `.gitlab-ci.yml` file: + +- The Git tag, description, and reference are created automatically in the pipeline. +- If you manually create a tag, the `release_job` job does not run. + +```yaml +release_job: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + when: never # Do not run this job when a tag is created manually + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch + script: + - echo "running release_job for $TAG" + release: # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties + tag_name: 'v0.$CI_PIPELINE_IID' # The version is incremented per pipeline. + description: 'v0.$CI_PIPELINE_IID' + ref: '$CI_COMMIT_SHA' # The tag is created from the pipeline SHA. +``` + +NOTE: +Environment variables set in `before_script` or `script` are not available for expanding +in the same job. Read more about +[potentially making variables available for expanding](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/6400). + +## Skip multiple pipelines when creating a release + +Creating a release using a CI/CD job could potentially trigger multiple pipelines if the associated tag does not exist already. To understand how this might happen, consider the following workflows: + +- Tag first, release second: + 1. A tag is created via UI or pushed. + 1. A tag pipeline is triggered, and runs `release` job. + 1. A release is created. + +- Release first, tag second: + 1. A pipeline is triggered when commits are pushed or merged to default branch. The pipeline runs `release` job. + 1. A release is created. + 1. A tag is created. + 1. A tag pipeline is triggered. The pipeline also runs `release` job. + +In the second workflow, the `release` job runs in multiple pipelines. To prevent this, you can use the [`workflow:rules` keyword](../../../ci/yaml/index.md#workflowrules) to determine if a release job should run in a tag pipeline: + +```yaml +release_job: + rules: + - if: $CI_COMMIT_TAG + when: never # Do not run this job in a tag pipeline + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run this job when commits are pushed or merged to the default branch + script: + - echo "Create release" + release: + name: 'My awesome release' + tag_name: '$CI_COMMIT_TAG' +``` diff --git a/doc/user/project/releases/release_cli.md b/doc/user/project/releases/release_cli.md index b55f0b0a734..9e65ab4bc01 100644 --- a/doc/user/project/releases/release_cli.md +++ b/doc/user/project/releases/release_cli.md @@ -4,16 +4,38 @@ group: Release info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Install the `release-cli` for the Shell executor **(FREE)** + +# GitLab Release CLI tool + +The [GitLab Release CLI (`release-cli`)](https://gitlab.com/gitlab-org/release-cli) tool +is a command-line tool for managing releases from the command line or from a CI/CD pipeline. +You can use the release CLI to create, update, modify, and delete releases. + +When you [use a CI/CD job to create a release](index.md#creating-a-release-by-using-a-cicd-job), +the `release` keyword entries are transformed into Bash commands and sent to the Docker +container containing the `release-cli` tool. The tool then creates the release. + +You can also call the `release-cli` tool directly from a [`script`](../../../ci/yaml/index.md#script). +For example: + +```shell +release-cli create --name "Release $CI_COMMIT_SHA" --description \ + "Created using the release-cli $EXTRA_DESCRIPTION" \ + --tag-name "v${MAJOR}.${MINOR}.${REVISION}" --ref "$CI_COMMIT_SHA" \ + --released-at "2020-07-15T08:00:00Z" --milestone "m1" --milestone "m2" --milestone "m3" \ + --assets-link "{\"name\":\"asset1\",\"url\":\"https://example.com/assets/1\",\"link_type\":\"other\"} +``` + +## Install the `release-cli` for the Shell executor **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/release-cli/-/issues/21) in GitLab 13.8. -> - [Changed](https://gitlab.com/gitlab-org/release-cli/-/merge_requests/108) in GitLab 14.2, the `release-cli` binaries are also [available in the Package Registry](https://gitlab.com/jaime/release-cli/-/packages). +> - [Changed](https://gitlab.com/gitlab-org/release-cli/-/merge_requests/108) in GitLab 14.2, the `release-cli` binaries are also [available in the Package Registry](https://gitlab.com/gitlab-org/release-cli/-/packages). When you use a runner with the Shell executor, you can download and install the `release-cli` manually for your [supported OS and architecture](https://release-cli-downloads.s3.amazonaws.com/latest/index.html). Once installed, [the `release` keyword](../../../ci/yaml/index.md#release) is available to use in your CI/CD jobs. -## Install on Unix/Linux +### Install on Unix/Linux 1. Download the binary for your system from S3, in the following example for amd64 systems: @@ -41,7 +63,7 @@ Once installed, [the `release` keyword](../../../ci/yaml/index.md#release) is av release-cli version 0.6.0 ``` -## Install on Windows PowerShell +### Install on Windows PowerShell 1. Create a folder somewhere in your system, for example `C:\GitLab\Release-CLI\bin` diff --git a/doc/user/project/releases/release_fields.md b/doc/user/project/releases/release_fields.md new file mode 100644 index 00000000000..647cac9c38e --- /dev/null +++ b/doc/user/project/releases/release_fields.md @@ -0,0 +1,274 @@ +--- +stage: Release +group: Release +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Release fields + +The following fields are available when you create or edit a release. + +## Title + +The release title can be customized using the **Release title** field when +creating or editing a release. If no title is provided, the release's tag name +is used instead. + +## Tag name + +The release tag name should include the release version. GitLab uses [Semantic Versioning](https://semver.org/) +for our releases, and we recommend you do too. Use `(Major).(Minor).(Patch)`, as detailed in the +[GitLab Policy for Versioning](../../../policy/maintenance.md#versioning). + +For example, for GitLab version `10.5.7`: + +- `10` represents the major version. The major release was `10.0.0`, but often referred to as `10.0`. +- `5` represents the minor version. The minor release was `10.5.0`, but often referred to as `10.5`. +- `7` represents the patch number. + +Any part of the version number can be multiple digits, for example, `13.10.11`. + +## Release notes description + +Every release has a description. You can add any text you like, but we recommend +including a changelog to describe the content of your release. This helps users +quickly scan the differences between each release you publish. + +[Git's tagging messages](https://git-scm.com/book/en/v2/Git-Basics-Tagging) can +be included in Release note descriptions by selecting **Include tag message in +the release notes**. + +Description supports [Markdown](../../markdown.md). + +## Release assets + +A release contains the following types of assets: + +- [Source code](#source-code) +- [Link](#links) + +### Source code + +GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar` +archived source code from the given Git tag. These are read-only assets. + +### Links + +A link is any URL which can point to whatever you like: documentation, built +binaries, or other related materials. These can be both internal or external +links from your GitLab instance. +Each link as an asset has the following attributes: + +| Attribute | Description | Required | +|-------------|--------------------------------------------------------------------------------------------------------------|----------| +| `name` | The name of the link. | Yes | +| `url` | The URL to download a file. | Yes | +| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No | +| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No | + +#### Permanent link to latest release + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. + +Latest release page is accessible through a permanent URL. +GitLab redirects to the latest release page URL when it is visited. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/permalink/latest +``` + +We also support, suffix path carry forward on the redirect to the latest release. +Example if release `v14.8.0-ee` is the latest release and has a readable link `https://host/namespace/project/-/releases/v14.8.0-ee#release` then it can be addressed as `https://host/namespace/project/-/releases/permalink/latest#release`. + +Refer [permanent links to latest release assets](#permanent-links-to-latest-release-assets) section to understand more about the suffix path carry forward usage. + +##### Sorting preferences + +By default, GitLab fetches the release using `released_at` time. The use of the query parameter `?order_by=released_at` is optional, and support for `?order_by=semver` is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352945). + +#### Permanent links to release assets + +The assets associated with a release are accessible through a permanent URL. +GitLab always redirects this URL to the actual asset +location, so even if the assets move to a different location, you can continue +to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link) using the `filepath` API attribute. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/:release/downloads/:filepath +``` + +If you have an asset for the `v11.9.0-rc2` release in the `gitlab-org` +namespace and `gitlab-runner` project on `gitlab.com`, for example: + +```json +{ + "name": "linux amd64", + "filepath": "/binaries/gitlab-runner-linux-amd64", + "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", + "link_type": "other" +} +``` + +This asset has a direct link of: + +```plaintext +https://gitlab.com/gitlab-org/gitlab-runner/-/releases/v11.9.0-rc2/downloads/binaries/gitlab-runner-linux-amd64 +``` + +The physical location of the asset can change at any time and the direct link remains unchanged. + +#### Permanent links to latest release assets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16821) in GitLab 14.9. + +The `filepath` from [permanent links to release assets](#permanent-links-to-release-assets) can be used in combination with [permanent link to the latest release](#permanent-link-to-latest-release). It is useful when we want to link a permanent URL to download an asset from the *latest release*. + +The format of the URL is: + +```plaintext +https://host/namespace/project/-/releases/permalink/latest/downloads/:filepath +``` + +If you have an asset with [`filepath`](../../../api/releases/links.md#create-a-link) for the `v11.9.0-rc2` latest release in the `gitlab-org` +namespace and `gitlab-runner` project on `gitlab.com`, for example: + +```json +{ + "name": "linux amd64", + "filepath": "/binaries/gitlab-runner-linux-amd64", + "url": "https://gitlab-runner-downloads.s3.amazonaws.com/v11.9.0-rc2/binaries/gitlab-runner-linux-amd64", + "link_type": "other" +} +``` + +This asset has a direct link of: + +```plaintext +https://gitlab.com/gitlab-org/gitlab-runner/-/releases/permalink/latest/downloads/binaries/gitlab-runner-linux-amd64 +``` + +#### Link Types + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1. + +The four types of links are "Runbook," "Package," "Image," and "Other." +The `link_type` parameter accepts one of the following four values: + +- `runbook` +- `package` +- `image` +- `other` (default) + +This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. + +#### Use a generic package for attaching binaries + +You can use [generic packages](../../packages/generic_packages/index.md) +to store any artifacts from a release or tag pipeline, +that can also be used for attaching binary files to an individual release entry. +You basically need to: + +1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file). +1. [Attach the package link to the release](#links). + +The following example generates release assets, publishes them +as a generic package, and then creates a release: + +```yaml +stages: + - build + - upload + - release + +variables: + # Package version can only contain numbers (0-9), and dots (.). + # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion. + # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file + PACKAGE_VERSION: "1.2.3" + DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}" + LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}" + PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}" + +build: + stage: build + image: alpine:latest + rules: + - if: $CI_COMMIT_TAG + script: + - mkdir bin + - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY} + - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY} + artifacts: + paths: + - bin/ + +upload: + stage: upload + image: curlimages/curl:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}" + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} "${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}" + +release: + # Caution, as of 2021-02-02 these assets links require a login, see: + # https://gitlab.com/gitlab-org/gitlab/-/issues/299384 + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ + --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \ + --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}" +``` + +PowerShell users may need to escape the double quote `"` inside a JSON +string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json` +before passing on to the `release-cli`. +For example: + +```yaml +release: + script: + - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}" + - $env:assetjson = $env:asset | ConvertTo-Json + - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson +``` + +NOTE: +Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md) +links to a release is not recommended, because artifacts are ephemeral and +are used to pass data in the same pipeline. This means there's a risk that +they could either expire or someone might manually delete them. + +### Number of new and total features **(FREE SAAS)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/235618) in GitLab 13.5. + +On [GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/releases), you can view the number of new and total features in the project. + + + +The totals are displayed on [shields](https://shields.io/) and are generated per release by +[a Rake task in the `www-gitlab-com` repository](https://gitlab.com/gitlab-com/www-gitlab-com/-/blob/master/lib/tasks/update_gitlab_project_releases_page.rake). + +| Item | Formula | +|------------------|------------------------------------------------------------------------------------| +| `New features` | Total count of release posts across all tiers for a single release in the project. | +| `Total features` | Total count of release posts in reverse order for all releases in the project. | + +The counts are also shown by license tier. + +| Item | Formula | +|------------------|-----------------------------------------------------------------------------------------------------| +| `New features` | Total count of release posts across a single tier for a single release in the project. | +| `Total features` | Total count of release posts across a single tier in reverse order for all releases in the project. | diff --git a/doc/user/project/repository/branches/default.md b/doc/user/project/repository/branches/default.md index 747da817195..3083ca5da3c 100644 --- a/doc/user/project/repository/branches/default.md +++ b/doc/user/project/repository/branches/default.md @@ -110,7 +110,7 @@ This setting applies only to each repository's default branch. To protect other you must either: - Configure [branch protection in the repository](../../../project/protected_branches.md). -- Configure [branch protection for groups](../../../group/index.md#change-the-default-branch-protection-of-a-group). +- Configure [branch protection for groups](../../../group/manage.md#change-the-default-branch-protection-of-a-group). Administrators of self-managed instances can customize the initial default branch protection for projects hosted on that instance. Individual groups and subgroups can override this instance-wide setting for their projects. diff --git a/doc/user/project/repository/forking_workflow.md b/doc/user/project/repository/forking_workflow.md index 85bea80f777..df75f19ea6c 100644 --- a/doc/user/project/repository/forking_workflow.md +++ b/doc/user/project/repository/forking_workflow.md @@ -26,7 +26,7 @@ To fork an existing project in GitLab: 1. On the project's home page, in the top right, select **{fork}** **Fork**:  1. Optional. Edit the **Project name**. -1. For **Project URL**, select the [namespace](../../group/index.md#namespaces) +1. For **Project URL**, select the [namespace](../../namespace/index.md) your fork should belong to. 1. Add a **Project slug**. This value becomes part of the URL to your fork. It must be unique in the namespace. diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index a8937d4f705..8e1286548b9 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -232,7 +232,7 @@ When a repository path changes, GitLab handles the transition from the old location to the new one with a redirect. When you [rename a user](../../profile/index.md#change-your-username), -[change a group path](../../group/index.md#change-a-groups-path), or [rename a repository](../settings/index.md#rename-a-repository): +[change a group path](../../group/manage.md#change-a-groups-path), or [rename a repository](../settings/index.md#rename-a-repository): - URLs for the namespace and everything under it, like projects, are redirected to the new URLs. diff --git a/doc/user/project/repository/managing_large_repositories.md b/doc/user/project/repository/managing_large_repositories.md index 93b94ac0641..ba425ae3dc7 100644 --- a/doc/user/project/repository/managing_large_repositories.md +++ b/doc/user/project/repository/managing_large_repositories.md @@ -16,6 +16,8 @@ On this page we detail several best practices to improve performance with these It's *strongly* recommended in any Git system that binary or blob files (for example, packages, audio, video, graphics, etc.) are stored as Large File Storage (LFS) objects. In such setup, the Objects are stored elsewhere, such as in Object Storage, and this can reduce the repository size significantly, thus improving performance. +To analyze if the repository has these sorts of objects, it's recommended to run [`git-sizer`](https://github.com/github/git-sizer) to get a detailed analysis. This tool shows in detail what makes up the repository as well as highlights any areas of concern. + Refer to the [Git LFS documentation for more information](../../../topics/git/lfs/index.md). ## Gitaly Pack Objects Cache @@ -32,7 +34,7 @@ In these types of setups it's recommended that the GitLab environment used match ## Gitaly Cluster -Gitaly Cluster can notably improve large repository performance as it holds multiple replicas of the repository across several nodes. As a result, Gitaly Cluster can load balance read requests against those repositories and is also fault tolerant. +Gitaly Cluster can notably improve large repository performance as it holds multiple replicas of the repository across several nodes. As a result, Gitaly Cluster can load balance read requests against those repositories and is also fault-tolerant. It's recommended for large repositories, however, Gitaly Cluster is a large solution with additional complexity of setup, and management. Refer to the [Gitaly Cluster documentation for more information](../../../administration/gitaly/index.md), specifically the [Before deploying Gitaly Cluster](../../../administration/gitaly/index.md#before-deploying-gitaly-cluster) section. diff --git a/doc/user/project/repository/mirror/index.md b/doc/user/project/repository/mirror/index.md index 4537f8520cd..b08530c34b3 100644 --- a/doc/user/project/repository/mirror/index.md +++ b/doc/user/project/repository/mirror/index.md @@ -17,8 +17,8 @@ Subscribe to the issue to follow its progress. Several mirroring methods exist: -- [Push](push.md): for mirroring a GitLab repository to another location. -- [Pull](pull.md): for mirroring a repository from another location to GitLab. +- [Push](push.md): Mirror a repository from GitLab to another location. +- [Pull](pull.md): Mirror a repository from another location to a GitLab Premium instance. - [Bidirectional](bidirectional.md) mirroring is also available, but can cause conflicts. Mirror a repository when: @@ -113,6 +113,11 @@ GitLab supports these authentication methods: - [SSH authentication](#ssh-authentication). - Password. +When using password authentication, ensure you specify the username. +For a [project access token](../../settings/project_access_tokens.md) or +[group access token](../../../group/settings/group_access_tokens.md), +use the username (not token name) and the token as the password. + ### SSH authentication SSH authentication is mutual: @@ -226,7 +231,7 @@ This error can occur when a firewall performs a `Deep SSH Inspection` on outgoin ### Could not read username: terminal prompts disabled If you receive this error after creating a new project using -[GitLab CI/CD for external repositories](../../../../ci/ci_cd_for_external_repos/): +[GitLab CI/CD for external repositories](../../../../ci/ci_cd_for_external_repos/index.md): - In Bitbucket Cloud: diff --git a/doc/user/project/repository/mirror/pull.md b/doc/user/project/repository/mirror/pull.md index 88104e34eb4..d0f2b9a8088 100644 --- a/doc/user/project/repository/mirror/pull.md +++ b/doc/user/project/repository/mirror/pull.md @@ -28,6 +28,11 @@ local repository, GitLab stops updating the branch. This prevents data loss. Deleted branches and tags in the upstream repository are not reflected in the downstream repository. +NOTE: +Items deleted from the downstream pull mirror repository, but still in the upstream repository, +are restored upon the next pull. For example: a branch deleted _only_ in the mirrored repository +reappears after the next pull. + ## How pull mirroring works After you configure a GitLab repository as a pull mirror: diff --git a/doc/user/project/repository/push_rules.md b/doc/user/project/repository/push_rules.md index 592aff85434..46a9585604e 100644 --- a/doc/user/project/repository/push_rules.md +++ b/doc/user/project/repository/push_rules.md @@ -25,7 +25,7 @@ For custom push rules use [server hooks](../../../administration/server_hooks.md ## Enable global push rules You can create push rules for all new projects to inherit, but they can be overridden -at the project level or the [group level](../../group/index.md#group-push-rules). +at the project level or the [group level](../../group/access_and_permissions.md#group-push-rules). All projects created after you configure global push rules inherit this configuration. However, each existing project must be updated manually, using the process described in [Override global push rules per project](#override-global-push-rules-per-project). diff --git a/doc/user/project/repository/reducing_the_repo_size_using_git.md b/doc/user/project/repository/reducing_the_repo_size_using_git.md index b0ae1b7d1e0..344c288b607 100644 --- a/doc/user/project/repository/reducing_the_repo_size_using_git.md +++ b/doc/user/project/repository/reducing_the_repo_size_using_git.md @@ -46,8 +46,8 @@ To purge files from a GitLab repository: [`git-sizer`](https://github.com/github/git-sizer#getting-started) using a supported package manager or from source. -1. Generate a fresh [export from the - project](../settings/import_export.html#export-a-project-and-its-data) and download it. +1. Generate a fresh + [export from the project](../settings/import_export.md#export-a-project-and-its-data) and download it. This project export contains a backup copy of your repository *and* refs we can use to purge files from your repository. @@ -195,8 +195,8 @@ When using repository cleanup, note: - Project statistics are cached. You may need to wait 5-10 minutes to see a reduction in storage utilization. - The cleanup prunes loose objects older than 30 minutes. This means objects added or referenced in the last 30 minutes - are not be removed immediately. If you have access to the - [Gitaly](../../../administration/gitaly/index.md) server, you may slip that delay and run `git gc --prune=now` to + are not removed immediately. If you have access to the + [Gitaly](../../../administration/gitaly/index.md) server, you may skip that delay and run `git gc --prune=now` to prune all loose objects immediately. - This process removes some copies of the rewritten commits from the GitLab cache and database, but there are still numerous gaps in coverage and some of the copies may persist indefinitely. @@ -207,7 +207,7 @@ When using repository cleanup, note: Repository size limits: -- Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md#account-and-limit-settings) +- Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md#account-and-limit-settings). - Can [be set by an administrator](../../admin_area/settings/account_and_limit_settings.md) on self-managed instances. - Are [set for GitLab.com](../../gitlab_com/index.md#account-and-limit-settings). diff --git a/doc/user/project/service_desk.md b/doc/user/project/service_desk.md index 17e55b7aac2..f32035102bb 100644 --- a/doc/user/project/service_desk.md +++ b/doc/user/project/service_desk.md @@ -307,8 +307,7 @@ In these issues, you can also see our friendly neighborhood [Support Bot](#suppo ### As an end user (issue creator) -> Support for additional email headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346600) in GitLab 14.6. -> In earlier versions, the Service Desk email address had to be in the "To" field. +> Support for additional email headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346600) in GitLab 14.6. In earlier versions, the Service Desk email address had to be in the "To" field. To create a Service Desk issue, an end user does not need to know anything about the GitLab instance. They just send an email to the address they are given, and diff --git a/doc/user/project/settings/index.md b/doc/user/project/settings/index.md index 7d1bfcaab59..b973a0f56d1 100644 --- a/doc/user/project/settings/index.md +++ b/doc/user/project/settings/index.md @@ -253,7 +253,7 @@ Use the toggles to enable or disable features in the project. | **Security & Compliance** | ✓ | Control access to [security features](../../application_security/index.md). | | **Wiki** | ✓ | Enables a separate system for [documentation](../wiki/). | | **Snippets** | ✓ | Enables [sharing of code and text](../../snippets.md). | -| **Pages** | ✓ | Allows you to [publish static websites](../pages/). | +| **Pages** | ✓ | Allows you to [publish static websites](../pages/index.md). | | **Operations** | ✓ | Control access to Operations-related features, including [Operations Dashboard](../../../operations/index.md), [Environments and Deployments](../../../ci/environments/index.md), [Feature Flags](../../../operations/feature_flags.md). | | **Metrics Dashboard** | ✓ | Control access to [metrics dashboard](../integrations/prometheus.md). | @@ -392,7 +392,7 @@ When you transfer a project to another namespace, you move the project to a diff Prerequisites: -- You must have at least the Maintainer role for the [group](../../group/index.md#create-a-group) to which you are transferring. +- You must have at least the Maintainer role for the [group](../../group/manage.md#create-a-group) to which you are transferring. - You must be the Owner of the project you transfer. - The group must allow creation of new projects. - The project must not contain any [container images](../../packages/container_registry/index.md#limitations). @@ -447,15 +447,16 @@ in GitLab 12.6, and then to [immediate deletion](https://gitlab.com/gitlab-org/g ### Delayed project deletion **(PREMIUM)** -> [Enabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89466) in GitLab 15.1. +> - [Enabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89466) in GitLab 15.1. +> - [Disabled for projects in personal namespaces](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95495) in GitLab 15.3. -Projects can be deleted after a delay period. Multiple settings can affect whether +Projects in a group (not a personal namespace) can be deleted after a delay period. Multiple settings can affect whether delayed project deletion is enabled for a particular project: - Self-managed instance [settings](../../admin_area/settings/visibility_and_access_controls.md#delayed-project-deletion). You can enable delayed project deletion as the default setting for new groups, and configure the number of days for the delay. For GitLab.com, see the [GitLab.com settings](../../gitlab_com/index.md#delayed-project-deletion). -- Group [settings](../../group/index.md#enable-delayed-project-deletion) to enabled delayed project deletion for all +- Group [settings](../../group/manage.md#enable-delayed-project-deletion) to enabled delayed project deletion for all projects in the group. ### Delete a project immediately diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index facaba45aec..5a4e300a210 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -16,15 +16,18 @@ Use the <kbd>.</kbd> [keyboard shortcut](../../shortcuts.md) to open the Web IDE You can also open the Web IDE when viewing a file, from the repository file list, and from merge requests: -- *When viewing a file, or the repository file list* - +### When viewing a file or the repository file list + 1. In the upper right corner of the page, select **Open in Web IDE** if it is visible. 1. If **Open in Web IDE** is not visible: 1. Select the (**{chevron-lg-down}**) next to **Edit** or **Gitpod**, depending on your configuration. 1. Select **Open in Web IDE** from the list to display it as the editing option. 1. Select **Open in Web IDE** to open the editor. -- *When viewing a merge request* - + +### When viewing a merge request + 1. Go to your merge request. - 1. In the upper right corner, select **Code**, then select **Open in Gitpod**. + 1. In the upper right corner, select **Code > Open in Web IDE**. ## File finder @@ -86,7 +89,7 @@ You can pick a theme from your [profile preferences](../../profile/preferences.m ## Highlight lines -WebIDE is built with the [Web Editor](../repository/web_editor.md). This enables WebIDE to share the +The Web IDE is built with the [Web Editor](../repository/web_editor.md). This enables the Web IDE to share the same core features for highlighting and linking to particular lines in the edited files [described for the Web Editor](../repository/web_editor.md#highlight-lines). diff --git a/doc/user/project/wiki/group.md b/doc/user/project/wiki/group.md index dc448fed970..a3ba5789d39 100644 --- a/doc/user/project/wiki/group.md +++ b/doc/user/project/wiki/group.md @@ -15,7 +15,7 @@ Group wikis are similar to [project wikis](index.md), with a few limitations: - [Git LFS](../../../topics/git/lfs/index.md) is not supported. - Group wikis are not included in [global search](../../search/advanced_search.md). -- Changes to group wikis don't show up in the [group's activity feed](../../group/index.md#group-activity-analytics). +- Changes to group wikis don't show up in the [group's activity feed](../../group/manage.md#group-activity-analytics). - Group wikis are enabled by default for GitLab Premium and higher tiers. You [can't turn them off from the GitLab user interface](https://gitlab.com/gitlab-org/gitlab/-/issues/208413). diff --git a/doc/user/project/wiki/index.md b/doc/user/project/wiki/index.md index 6e320923496..c7f675417bb 100644 --- a/doc/user/project/wiki/index.md +++ b/doc/user/project/wiki/index.md @@ -6,6 +6,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Wiki **(FREE)** +> - Page loading [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/336792) to asynchronous in GitLab 14.9. +> - Page slug encoding method [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71753) to `ERB::Util.url_encode` in GitLab 14.9. + If you don't want to keep your documentation in your repository, but you want to keep it in the same project as your code, you can use the wiki GitLab provides in each GitLab project. Every wiki is a separate Git repository, so you can create @@ -230,7 +233,7 @@ GitLab tracks wiki creation, deletion, and update events. These events are displ - [User profile](../../profile/index.md#access-your-user-profile). - Activity pages, depending on the type of wiki: - - [Group activity](../../group/index.md#view-group-activity). + - [Group activity](../../group/manage.md#view-group-activity). - [Project activity](../working_with_projects.md#view-project-activity). Commits to wikis are not counted in [repository analytics](../../analytics/repository_analytics.md). @@ -337,7 +340,7 @@ Support includes: - List formatting for unordered, numbered, and checklists. - Creating and editing the structure of tables. - Inserting and formatting code blocks with syntax highlighting. -- Live preview of Mermaid, PlantUML, and Kroki diagrams ([Introduced]<https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86701> in GitLab 15.2). +- Live preview of Mermaid, PlantUML, and Kroki diagrams ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86701) in GitLab 15.2). ### Use the Content Editor @@ -370,3 +373,35 @@ For the status of the ongoing development for CommonMark and GitLab Flavored Mar - [Group repository storage moves API](../../../api/group_repository_storage_moves.md) - [Group wikis API](../../../api/group_wikis.md) - [Wiki keyboard shortcuts](../../shortcuts.md#wiki-pages) + +## Troubleshooting + +### Page slug rendering with Apache reverse proxy + +In GitLab 14.9 and later, page slugs are now encoded using the +[`ERB::Util.url_encode`](https://www.rubydoc.info/stdlib/erb/ERB%2FUtil.url_encode) method. +If you use an Apache reverse proxy, you can add a `nocanon` argument to the `ProxyPass` +line of your Apache configuration to ensure your page slugs render correctly. + +### Recreate a project wiki with the Rails console **(FREE SELF)** + +WARNING: +This operation deletes all data in the wiki. + +To clear all data from a project wiki and recreate it in a blank state: + +1. [Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session). +1. Run these commands: + + ```ruby + # Enter your project's path + p = Project.find_by_full_path('<username-or-group>/<project-name>') + + # This command deletes the wiki project from the filesystem. + GitlabShellWorker.perform_in(0, :remove_repository, p.repository_storage, p.wiki.disk_path) + + # Refresh the wiki repository state. + p.wiki.repository.expire_exists_cache + ``` + +All data from the wiki has been cleared, and the wiki is ready for use. diff --git a/doc/user/project/working_with_projects.md b/doc/user/project/working_with_projects.md index 9572bc241fc..2501fa8b45c 100644 --- a/doc/user/project/working_with_projects.md +++ b/doc/user/project/working_with_projects.md @@ -233,7 +233,7 @@ To push your repository and create a project: ``` - For `gitlab.example.com`, use the domain name of the machine that hosts your Git repository. - - For `namespace`, use the name of your [namespace](../group/index.md#namespaces). + - For `namespace`, use the name of your [namespace](../namespace/index.md). - For `myproject`, use the name of your project. - Optional. To export existing repository tags, append the `--tags` flag to your `git push` command. 1. Optional. To configure the remote: @@ -290,7 +290,7 @@ To view your personal projects: ## Delete a project After you delete a project, projects in personal namespaces are deleted immediately. To delay deletion of projects in a group -you can [enable delayed project removal](../group/index.md#enable-delayed-project-deletion). +you can [enable delayed project removal](../group/manage.md#enable-delayed-project-deletion). To delete a project: @@ -310,7 +310,7 @@ To delete a project: > - [Available to all users](https://gitlab.com/gitlab-org/gitlab/-/issues/346976) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `project_owners_list_project_pending_deletion`. Enabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/351556) in GitLab 14.9. [Feature flag `project_owners_list_project_pending_deletion`](https://gitlab.com/gitlab-org/gitlab/-/issues/351556) removed. -When delayed project deletion is [enabled for a group](../group/index.md#enable-delayed-project-deletion), +When delayed project deletion is [enabled for a group](../group/manage.md#enable-delayed-project-deletion), projects within that group are not deleted immediately, but only after a delay. To view a list of all projects that are pending deletion: @@ -371,7 +371,7 @@ To leave a project: 1. Select a project. 1. Select **Leave project**. The **Leave project** option only displays on the project dashboard when a project is part of a group under a -[group namespace](../group/index.md#namespaces). +[group namespace](../namespace/index.md). ## Use a project as a Go package diff --git a/doc/user/ssh.md b/doc/user/ssh.md index e884d762379..5667890757a 100644 --- a/doc/user/ssh.md +++ b/doc/user/ssh.md @@ -304,8 +304,10 @@ Verify that your SSH key was added correctly. The following commands use the example hostname `gitlab.example.com`. Replace this example hostname with your GitLab instance's hostname, for example, `git@gitlab.com`. -1. For GitLab.com, to ensure you're connecting to the correct server, confirm the - [SSH host keys fingerprints](gitlab_com/index.md#ssh-host-keys-fingerprints). +1. To ensure you're connecting to the correct server, check the server's SSH host keys fingerprint. For: + - GitLab.com, see the [SSH host keys fingerprints](gitlab_com/index.md#ssh-host-keys-fingerprints) documentation. + - GitLab.com or another GitLab instance, see `gitlab.example.com/help/instance_configuration#ssh-host-keys-fingerprints` where `gitlab.example.com` is `gitlab.com` (for + GitLab.com) or the address of the GitLab instance. 1. Open a terminal and run this command, replacing `gitlab.example.com` with your GitLab instance URL: diff --git a/doc/user/tasks.md b/doc/user/tasks.md index 36236f2969e..b5c2c4eb3a5 100644 --- a/doc/user/tasks.md +++ b/doc/user/tasks.md @@ -8,12 +8,22 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334812) in GitLab 14.5 [with a flag](../administration/feature_flags.md) named `work_items`. Disabled by default. > - [Creating, editing, and deleting tasks](https://gitlab.com/groups/gitlab-org/-/epics/7169) introduced in GitLab 15.0. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/334812) in GitLab 15.3. + +WARNING: +Tasks are in [**Alpha**](../policy/alpha-beta-support.md#alpha-features). + +The following list are the known limitations: + +- [Tasks currently cannot be accessed via REST API.](https://gitlab.com/gitlab-org/gitlab/-/issues/368055) +- An issue's tasks can only currently be accessed via a reference within a description, comment, or direct URL (`.../-/work_items/[global_id]`). + +For the latest updates, check the [Tasks Roadmap](https://gitlab.com/groups/gitlab-org/-/epics/7103). FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `work_items`. -On GitLab.com, this feature is not available. -The feature is not ready for production use. +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flags](../administration/feature_flags.md) named `work_items` and `work_items_hierarchy`. +On GitLab.com, this feature is available. Use tasks to track steps needed for the [issue](project/issues/index.md) to be closed. diff --git a/doc/user/upgrade_email_bypass.md b/doc/user/upgrade_email_bypass.md index 6401828270d..1b267a0569b 100644 --- a/doc/user/upgrade_email_bypass.md +++ b/doc/user/upgrade_email_bypass.md @@ -83,7 +83,7 @@ You have the following options to help your users: - They can confirm their address through the email that they received. - They can confirm the subject email address themselves by navigating to `https://gitlab.example.com/users/confirmation/new`. -As an administrator, you may also confirm a user in the [Admin Area](admin_area/#administering-users). +As an administrator, you may also confirm a user in the [Admin Area](admin_area/index.md#administering-users). ## What do I do if I am an administrator and I am locked out? diff --git a/doc/user/usage_quotas.md b/doc/user/usage_quotas.md index c863a9d8270..5d78b4bb795 100644 --- a/doc/user/usage_quotas.md +++ b/doc/user/usage_quotas.md @@ -12,8 +12,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Namespace storage limit -Namespaces on a GitLab SaaS Free tier have a 5 GB storage limit. For more information, see our [pricing page](https://about.gitlab.com/pricing/). -This limit is not visible on the storage quota page, but we plan to make it visible and enforced starting October 19, 2022. +Namespaces on GitLab SaaS have a storage limit. For more information, see our [pricing page](https://about.gitlab.com/pricing/). +This limit is not visible on the Usage quotas page, but will be prior to [enforcement](#namespace-storage-limit-enforcement-schedule). Self-managed deployments are not affected. Storage types that add to the total namespace storage are: @@ -22,7 +22,7 @@ Storage types that add to the total namespace storage are: - Artifacts - Container registry - Package registry -- Dependecy proxy +- Dependency proxy - Wiki - Snippets @@ -30,27 +30,18 @@ If your total namespace storage exceeds the available namespace storage quota, a To prevent exceeding the namespace storage quota, you can: -1. [Purchase more storage](../subscriptions/gitlab_com/index.md#purchase-more-storage-and-transfer). -1. [Upgrade to a paid tier](../subscriptions/gitlab_com/#upgrade-your-gitlab-saas-subscription-tier). -1. [Reduce storage usage](#manage-your-storage-usage). +1. Reduce storage consumption by following the suggestions in the [Manage Your Storage Usage](#manage-your-storage-usage) section of this page. +1. Apply for [GitLab for Education](https://about.gitlab.com/solutions/education/join/), [GitLab for Open Source](https://about.gitlab.com/solutions/open-source/join/), or [GitLab for Startups](https://about.gitlab.com/solutions/startups/) if you meet the eligibility requirements. +1. Consider using a [self-managed instance](../subscriptions/self_managed/) of GitLab which does not have these limits on the free tier. +1. [Purchase additional storage](../subscriptions/gitlab_com/index.md#purchase-more-storage-and-transfer) units at $60/year for 10GB of storage. +1. [Start a trial](https://about.gitlab.com/free-trial/) or [upgrade to GitLab Premium or Ultimate](https://about.gitlab.com/pricing) which include higher limits and features that enable growing teams to ship faster without sacrificing on quality. +1. [Talk to an expert](https://page.gitlab.com/usage_limits_help.html) to learn more about your options and ask questions. ### Namespace storage limit enforcement schedule -Starting October 19, 2022, a storage limit will be enforced on all GitLab Free namespaces. -We will start with a large limit enforcement and eventually reduce it to 5 GB. +Storage limits for GitLab SaaS Free tier namespaces will not be enforced prior to 2022-10-19. Storage limits for GitLab SaaS Paid tier namespaces will not be enforced for prior to 2023-02-15. -The following table describes the enforcement schedule, which is subject to change. - -| Target enforcement date | Limit | Expected Impact | Status | -| ------ | ------ | ------ | ------ | -| October 19, 2022 | 45,000 GB | LOW | Pending (**{hourglass}**)| -| October 20, 2022 | 7,500 GB | LOW | Pending (**{hourglass}**)| -| October 24, 2022 | 500 GB | MEDIUM | Pending (**{hourglass}**)| -| October 27, 2022 | 75 GB | MEDIUM HIGH | Pending (**{hourglass}**)| -| November 2, 2022 | 10 GB | HIGH | Pending (**{hourglass}**)| -| November 9, 2022 | 5 GB | VERY HIGH | Pending (**{hourglass}**)| - -Namespaces that reach the enforced limit will have their projects locked. To unlock your project, you will have to [manage its storage](#manage-your-storage-usage). +Impacted users are notified via email and in-app notifications at least 60 days prior to enforcement. ### Project storage limit @@ -67,7 +58,7 @@ you must purchase additional storage. For more details, see [Excess storage usag ## View storage usage -You can view storage usage for your project or [namespace](../user/group/#namespaces). +You can view storage usage for your project or [namespace](../user/namespace/index.md). 1. Go to your project or namespace: - For a project, on the top bar, select **Menu > Projects** and find your project. diff --git a/doc/user/workspace/index.md b/doc/user/workspace/index.md index 0ef3eb8b6fe..1c5a77c1f0c 100644 --- a/doc/user/workspace/index.md +++ b/doc/user/workspace/index.md @@ -17,14 +17,14 @@ sole discretion of GitLab Inc. NOTE: Workspace is currently in development. -Workspace will be above the [top-level namespaces](../group/index.md#namespaces) for you to manage +Workspace will be above the [top-level namespaces](../namespace/index.md) for you to manage everything you do as a GitLab administrator, including: - Defining and applying settings to all of your groups, subgroups, and projects. - Aggregating data from all your groups, subgroups, and projects. Our goal is to reach feature parity between SaaS and self-managed installations, with all -[Admin Area settings](/ee/user/admin_area/settings/) moving to either: +[Admin Area settings](/ee/user/admin_area/settings/index.md) moving to either: - Groups. Available in the Workspace, top-level group namespaces, and sub-groups. - Hardware Controls. For functionality that does not apply to groups, Hardware Controls are only |
